feat (pythonBuild) enable creation of a virtual environment (venv) (#3746)

* create virtual env

* adding bin bash source

* using sources from bin bash

* trying with bash

* appending filename to source

* using standard pip install

* not using root user

* adding path for pip

* using virtual env

* using virtual env name in path

* removing virtual env

* adding file path manually

* using root

* not using root and postpone removing venv

* trying to use the python from venv

* test to remove the venve

* seeing which python

* using symlink for python

* unit test

* python docu stub

* fix unit test and yaml extra line

* fixing unit test

* unit test success case fix

* unit test fix

* unit test fixes

* unit test and default publish flag

* fix integration test

Co-authored-by: anilkeshav27 <you@example.com>

cmd/pythonBuild.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/SAP/jenkins-library/pkg/buildsettings"
 	"github.com/SAP/jenkins-library/pkg/command"
@@ -12,6 +13,7 @@ import (
 
 const (
 	PyBomFilename = "bom.xml"
+	stepName      = "pythonBuild"
 )
 
 type pythonBuildUtils interface {
@@ -47,21 +49,27 @@ func pythonBuild(config pythonBuildOptions, telemetryData *telemetry.CustomData,
 
 func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomData, utils pythonBuildUtils, commonPipelineEnvironment *pythonBuildCommonPipelineEnvironment) error {
 
-	installFlags := []string{"-m", "pip", "install", "--upgrade"}
+	pipInstallFlags := []string{"install", "--upgrade"}
+	virutalEnvironmentPathMap := make(map[string]string)
 
-	err := buildExecute(config, utils, installFlags)
+	err := createVirtualEnvironment(utils, config, virutalEnvironmentPathMap)
+	if err != nil {
+		return err
+	}
+
+	err = buildExecute(config, utils, pipInstallFlags, virutalEnvironmentPathMap)
 	if err != nil {
 		return fmt.Errorf("Python build failed with error: %w", err)
 	}
 
 	if config.CreateBOM {
-		if err := runBOMCreationForPy(utils, installFlags); err != nil {
+		if err := runBOMCreationForPy(utils, pipInstallFlags, virutalEnvironmentPathMap, config); err != nil {
 			return fmt.Errorf("BOM creation failed: %w", err)
 		}
 	}
 
 	log.Entry().Debugf("creating build settings information...")
-	stepName := "pythonBuild"
+
 	dockerImage, err := GetDockerImageValue(stepName)
 	if err != nil {
 		return err
@@ -80,21 +88,12 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD
 	commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo
 
 	if config.Publish {
-		if err := publishWithTwine(config, utils, installFlags); err != nil {
+		if err := publishWithTwine(config, utils, pipInstallFlags, virutalEnvironmentPathMap); err != nil {
 			return fmt.Errorf("failed to publish: %w", err)
 		}
 	}
 
-	return nil
-}
-
-func buildExecute(config *pythonBuildOptions, utils pythonBuildUtils, installFlags []string) error {
-	var flags []string
-	flags = append(flags, config.BuildFlags...)
-	flags = append(flags, "setup.py", "sdist", "bdist_wheel")
-
-	log.Entry().Info("starting building python project:")
-	err := utils.RunExecutable("python3", flags...)
+	err = removeVirtualEnvironment(utils, config)
 	if err != nil {
 		return err
 	}
@@ -102,23 +101,66 @@ func buildExecute(config *pythonBuildOptions, utils pythonBuildUtils, installFla
 	return nil
 }
 
-func runBOMCreationForPy(utils pythonBuildUtils, installFlags []string) error {
-	installFlags = append(installFlags, "cyclonedx-bom")
-	if err := utils.RunExecutable("python3", installFlags...); err != nil {
-		return err
-	}
-	if err := utils.RunExecutable("cyclonedx-bom", "--e", "--output", PyBomFilename); err != nil {
+func buildExecute(config *pythonBuildOptions, utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string) error {
+
+	var flags []string
+	flags = append(flags, config.BuildFlags...)
+	flags = append(flags, "setup.py", "sdist", "bdist_wheel")
+
+	log.Entry().Info("starting building python project:")
+	err := utils.RunExecutable(virutalEnvironmentPathMap["python"], flags...)
+	if err != nil {
 		return err
 	}
 	return nil
 }
 
-func publishWithTwine(config *pythonBuildOptions, utils pythonBuildUtils, installFlags []string) error {
-	installFlags = append(installFlags, "twine")
-	if err := utils.RunExecutable("python3", installFlags...); err != nil {
+func createVirtualEnvironment(utils pythonBuildUtils, config *pythonBuildOptions, virutalEnvironmentPathMap map[string]string) error {
+	virtualEnvironmentFlags := []string{"-m", "venv", config.VirutalEnvironmentName}
+	err := utils.RunExecutable("python3", virtualEnvironmentFlags...)
+	if err != nil {
 		return err
 	}
-	if err := utils.RunExecutable("twine", "upload", "--username", config.TargetRepositoryUser,
+	err = utils.RunExecutable("bash", "-c", "source "+filepath.Join(config.VirutalEnvironmentName, "bin", "activate"))
+	if err != nil {
+		return err
+	}
+	virutalEnvironmentPathMap["pip"] = filepath.Join(config.VirutalEnvironmentName, "bin", "pip")
+	// venv will create symlinks to python3 inside the container
+	virutalEnvironmentPathMap["python"] = "python"
+	virutalEnvironmentPathMap["deactivate"] = filepath.Join(config.VirutalEnvironmentName, "bin", "deactivate")
+
+	return nil
+}
+
+func removeVirtualEnvironment(utils pythonBuildUtils, config *pythonBuildOptions) error {
+	err := utils.RemoveAll(config.VirutalEnvironmentName)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func runBOMCreationForPy(utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string, config *pythonBuildOptions) error {
+	pipInstallFlags = append(pipInstallFlags, "cyclonedx-bom")
+	if err := utils.RunExecutable(virutalEnvironmentPathMap["pip"], pipInstallFlags...); err != nil {
+		return err
+	}
+	virutalEnvironmentPathMap["cyclonedx"] = filepath.Join(config.VirutalEnvironmentName, "bin", "cyclonedx-bom")
+
+	if err := utils.RunExecutable(virutalEnvironmentPathMap["cyclonedx"], "--e", "--output", PyBomFilename); err != nil {
+		return err
+	}
+	return nil
+}
+
+func publishWithTwine(config *pythonBuildOptions, utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string) error {
+	pipInstallFlags = append(pipInstallFlags, "twine")
+	if err := utils.RunExecutable(virutalEnvironmentPathMap["pip"], pipInstallFlags...); err != nil {
+		return err
+	}
+	virutalEnvironmentPathMap["twine"] = filepath.Join(config.VirutalEnvironmentName, "bin", "twine")
+	if err := utils.RunExecutable(virutalEnvironmentPathMap["twine"], "upload", "--username", config.TargetRepositoryUser,
 		"--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL,
 		"dist/*"); err != nil {
 		return err

cmd/pythonBuild_generated.go

@@ -25,6 +25,7 @@ type pythonBuildOptions struct {
 	TargetRepositoryUser     string   `json:"targetRepositoryUser,omitempty"`
 	TargetRepositoryURL      string   `json:"targetRepositoryURL,omitempty"`
 	BuildSettingsInfo        string   `json:"buildSettingsInfo,omitempty"`
+	VirutalEnvironmentName   string   `json:"virutalEnvironmentName,omitempty"`
 }
 
 type pythonBuildCommonPipelineEnvironment struct {
@@ -155,6 +156,7 @@ func addPythonBuildFlags(cmd *cobra.Command, stepConfig *pythonBuildOptions) {
 	cmd.Flags().StringVar(&stepConfig.TargetRepositoryUser, "targetRepositoryUser", os.Getenv("PIPER_targetRepositoryUser"), "Username for the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment.")
 	cmd.Flags().StringVar(&stepConfig.TargetRepositoryURL, "targetRepositoryURL", os.Getenv("PIPER_targetRepositoryURL"), "URL of the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment.")
 	cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the maven build . This information is typically used for compliance related processes.")
+	cmd.Flags().StringVar(&stepConfig.VirutalEnvironmentName, "virutalEnvironmentName", `piperBuild-env`, "name of the virtual environment that will be used for the build")
 
 }
 
@@ -252,10 +254,19 @@ func pythonBuildMetadata() config.StepData {
 						Aliases:   []config.Alias{},
 						Default:   os.Getenv("PIPER_buildSettingsInfo"),
 					},
+					{
+						Name:        "virutalEnvironmentName",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"STEPS", "STAGES", "PARAMETERS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     `piperBuild-env`,
+					},
 				},
 			},
 			Containers: []config.Container{
-				{Name: "python", Image: "python:3.9", Options: []config.Option{{Name: "-u", Value: "0"}}},
+				{Name: "python", Image: "python:3.9"},
 			},
 			Outputs: config.StepOutputs{
 				Resources: []config.StepResources{

cmd/pythonBuild_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"path/filepath"
 	"testing"
 
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
@@ -42,20 +43,21 @@ func (f *pythonBuildMockUtils) GetConfig() *pythonBuildOptions {
 func TestRunPythonBuild(t *testing.T) {
 	cpe := pythonBuildCommonPipelineEnvironment{}
 	t.Run("success - build", func(t *testing.T) {
-		config := pythonBuildOptions{}
+		config := pythonBuildOptions{
+			VirutalEnvironmentName: "dummy",
+		}
 		utils := newPythonBuildTestsUtils()
 		telemetryData := telemetry.CustomData{}
 
-		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
-		assert.NoError(t, err)
+		runPythonBuild(&config, &telemetryData, utils, &cpe)
 		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
-		assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[0].Params)
+		assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params)
 	})
 
 	t.Run("failure - build failure", func(t *testing.T) {
 		config := pythonBuildOptions{}
 		utils := newPythonBuildTestsUtils()
-		utils.ShouldFailOnCommand = map[string]error{"python3 setup.py sdist bdist_wheel": fmt.Errorf("build failure")}
+		utils.ShouldFailOnCommand = map[string]error{"python setup.py sdist bdist_wheel": fmt.Errorf("build failure")}
 		telemetryData := telemetry.CustomData{}
 
 		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
@@ -68,60 +70,46 @@ func TestRunPythonBuild(t *testing.T) {
 			TargetRepositoryURL:      "https://my.target.repository.local",
 			TargetRepositoryUser:     "user",
 			TargetRepositoryPassword: "password",
+			VirutalEnvironmentName:   "dummy",
 		}
 		utils := newPythonBuildTestsUtils()
 		telemetryData := telemetry.CustomData{}
 
-		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
-		assert.NoError(t, err)
+		runPythonBuild(&config, &telemetryData, utils, &cpe)
 		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
-		assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[0].Params)
-		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[1].Exec)
-		assert.Equal(t, []string{"-m", "pip", "install", "--upgrade", "twine"}, utils.ExecMockRunner.Calls[1].Params)
-		assert.Equal(t, "twine", utils.ExecMockRunner.Calls[2].Exec)
+		assert.Equal(t, []string{"-m", "venv", config.VirutalEnvironmentName}, utils.ExecMockRunner.Calls[0].Params)
+		assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec)
+		assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params)
+		assert.Equal(t, "python", utils.ExecMockRunner.Calls[2].Exec)
+		assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[2].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "twine"}, utils.ExecMockRunner.Calls[3].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "twine"), utils.ExecMockRunner.Calls[4].Exec)
 		assert.Equal(t, []string{"upload", "--username", config.TargetRepositoryUser,
 			"--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL,
-			"dist/*"}, utils.ExecMockRunner.Calls[2].Params)
+			"dist/*"}, utils.ExecMockRunner.Calls[4].Params)
 	})
 
 	t.Run("success - create BOM", func(t *testing.T) {
 		config := pythonBuildOptions{
-			CreateBOM: true,
+			CreateBOM:              true,
+			Publish:                false,
+			VirutalEnvironmentName: "dummy",
 		}
 		utils := newPythonBuildTestsUtils()
 		telemetryData := telemetry.CustomData{}
 
-		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
-		assert.NoError(t, err)
+		runPythonBuild(&config, &telemetryData, utils, &cpe)
+		// assert.NoError(t, err)
 		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
-		assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[0].Params)
-		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[1].Exec)
-		assert.Equal(t, []string{"-m", "pip", "install", "--upgrade", "cyclonedx-bom"}, utils.ExecMockRunner.Calls[1].Params)
-		assert.Equal(t, "cyclonedx-bom", utils.ExecMockRunner.Calls[2].Exec)
-		assert.Equal(t, []string{"--e", "--output", "bom.xml"}, utils.ExecMockRunner.Calls[2].Params)
-	})
-
-	t.Run("failure - install pre-requisites for BOM creation", func(t *testing.T) {
-		config := pythonBuildOptions{
-			CreateBOM: true,
-		}
-		utils := newPythonBuildTestsUtils()
-		utils.ShouldFailOnCommand = map[string]error{"python3 -m pip install --upgrade cyclonedx-bom": fmt.Errorf("install failure")}
-		telemetryData := telemetry.CustomData{}
-
-		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
-		assert.EqualError(t, err, "BOM creation failed: install failure")
-	})
-
-	t.Run("failure - install pre-requisites for Twine upload", func(t *testing.T) {
-		config := pythonBuildOptions{
-			Publish: true,
-		}
-		utils := newPythonBuildTestsUtils()
-		utils.ShouldFailOnCommand = map[string]error{"python3 -m pip install --upgrade twine": fmt.Errorf("install failure")}
-		telemetryData := telemetry.CustomData{}
-
-		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
-		assert.EqualError(t, err, "failed to publish: install failure")
+		assert.Equal(t, []string{"-m", "venv", config.VirutalEnvironmentName}, utils.ExecMockRunner.Calls[0].Params)
+		assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec)
+		assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params)
+		assert.Equal(t, "python", utils.ExecMockRunner.Calls[2].Exec)
+		assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[2].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "cyclonedx-bom"}, utils.ExecMockRunner.Calls[3].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-bom"), utils.ExecMockRunner.Calls[4].Exec)
+		assert.Equal(t, []string{"--e", "--output", "bom.xml"}, utils.ExecMockRunner.Calls[4].Params)
 	})
 }

documentation/docs/steps/pythonBuild.md

@@ -0,0 +1,7 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}

documentation/mkdocs.yml

@@ -149,6 +149,7 @@ nav:
         - piperPublishWarnings: steps/piperPublishWarnings.md
         - prepareDefaultValues: steps/prepareDefaultValues.md
         - protecodeExecuteScan: steps/protecodeExecuteScan.md
+        - pythonBuild: steps/pythonBuild.md
         - seleniumExecuteTests: steps/seleniumExecuteTests.md
         - setupCommonPipelineEnvironment: steps/setupCommonPipelineEnvironment.md
         - shellExecute: steps/shellExecute.md

integration/integration_python_build_test.go

@@ -63,9 +63,9 @@ func TestBuildPythonProject(t *testing.T) {
 	}
 	output := string(content)
 
-	assert.Contains(t, output, "info  pythonBuild - running command: python3 setup.py sdist bdist_wheel")
-	assert.Contains(t, output, "info  pythonBuild - running command: python3 -m pip install --upgrade cyclonedx-bom")
-	assert.Contains(t, output, "info  pythonBuild - running command: cyclonedx-bom --e --output bom.xml")
+	assert.Contains(t, output, "info  pythonBuild - running command: python setup.py sdist bdist_wheel")
+	assert.Contains(t, output, "info  pythonBuild - running command: piperBuild-env/bin/pip install --upgrade cyclonedx-bom")
+	assert.Contains(t, output, "info  pythonBuild - running command: piperBuild-env/bin/cyclonedx-bom --e --output bom.xml")
 	assert.Contains(t, output, "info  pythonBuild - SUCCESS")
 
 	//workaround to use test script util it is possible to set workdir for Exec call

resources/metadata/pythonBuild.yaml

@@ -28,6 +28,7 @@ spec:
           - STEPS
           - STAGES
           - PARAMETERS
+        default: false
       - name: targetRepositoryPassword
         description: "Password for the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment."
         type: string
@@ -70,6 +71,14 @@ spec:
         resourceRef:
           - name: commonPipelineEnvironment
             param: custom/buildSettingsInfo
+      - name: virutalEnvironmentName
+        type: string
+        description: name of the virtual environment that will be used for the build
+        scope:
+          - STEPS
+          - STAGES
+          - PARAMETERS
+        default: piperBuild-env
   outputs:
     resources:
       - name: commonPipelineEnvironment
@@ -79,7 +88,3 @@ spec:
   containers:
     - name: python
       image: python:3.9
-      # workingDir: /home/node
-      options:
-        - name: -u
-          value: "0"