jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-01-18 05:18:24 +02:00
Code Issues Releases Activity

feat (pythonBuild) enable creation of a virtual environment (venv) (#3746)

Browse Source 
* create virtual env

* adding bin bash source

* using sources from bin bash

* trying with bash

* appending filename to source

* using standard pip install

* not using root user

* adding path for pip

* using virtual env

* using virtual env name in path

* removing virtual env

* adding file path manually

* using root

* not using root and postpone removing venv

* trying to use the python from venv

* test to remove the venve

* seeing which python

* using symlink for python

* unit test

* python docu stub

* fix unit test and yaml extra line

* fixing unit test

* unit test success case fix

* unit test fix

* unit test fixes

* unit test and default publish flag

* fix integration test

Co-authored-by: anilkeshav27 <you@example.com>
This commit is contained in:
Anil Keshav 2022-04-26 14:15:24 +02:00 committed by GitHub
parent 0696db5e0d
commit 1272b763f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 131 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
cmd/pythonBuild.go
View File

@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"path/filepath"
"github.com/SAP/jenkins-library/pkg/buildsettings"
"github.com/SAP/jenkins-library/pkg/command"
@ -12,6 +13,7 @@ import (
const (
PyBomFilename = "bom.xml"
stepName = "pythonBuild"
)
type pythonBuildUtils interface {
@ -47,21 +49,27 @@ func pythonBuild(config pythonBuildOptions, telemetryData *telemetry.CustomData,
func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomData, utils pythonBuildUtils, commonPipelineEnvironment *pythonBuildCommonPipelineEnvironment) error {
installFlags := []string{"-m", "pip", "install", "--upgrade"}
pipInstallFlags := []string{"install", "--upgrade"}
virutalEnvironmentPathMap := make(map[string]string)
err := buildExecute(config, utils, installFlags)
err := createVirtualEnvironment(utils, config, virutalEnvironmentPathMap)
if err != nil {
return err
}
err = buildExecute(config, utils, pipInstallFlags, virutalEnvironmentPathMap)
if err != nil {
return fmt.Errorf("Python build failed with error: %w", err)
}
if config.CreateBOM {
if err := runBOMCreationForPy(utils, installFlags); err != nil {
if err := runBOMCreationForPy(utils, pipInstallFlags, virutalEnvironmentPathMap, config); err != nil {
return fmt.Errorf("BOM creation failed: %w", err)
}
}
log.Entry().Debugf("creating build settings information...")
stepName := "pythonBuild"
dockerImage, err := GetDockerImageValue(stepName)
if err != nil {
return err
@ -80,21 +88,12 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD
commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo
if config.Publish {
if err := publishWithTwine(config, utils, installFlags); err != nil {
if err := publishWithTwine(config, utils, pipInstallFlags, virutalEnvironmentPathMap); err != nil {
return fmt.Errorf("failed to publish: %w", err)
}
}
return nil
}
func buildExecute(config *pythonBuildOptions, utils pythonBuildUtils, installFlags []string) error {
var flags []string
flags = append(flags, config.BuildFlags...)
flags = append(flags, "setup.py", "sdist", "bdist_wheel")
log.Entry().Info("starting building python project:")
err := utils.RunExecutable("python3", flags...)
err = removeVirtualEnvironment(utils, config)
if err != nil {
return err
}
@ -102,23 +101,66 @@ func buildExecute(config *pythonBuildOptions, utils pythonBuildUtils, installFla
return nil
}
func runBOMCreationForPy(utils pythonBuildUtils, installFlags []string) error {
installFlags = append(installFlags, "cyclonedx-bom")
if err := utils.RunExecutable("python3", installFlags...); err != nil {
return err
}
if err := utils.RunExecutable("cyclonedx-bom", "--e", "--output", PyBomFilename); err != nil {
func buildExecute(config *pythonBuildOptions, utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string) error {
var flags []string
flags = append(flags, config.BuildFlags...)
flags = append(flags, "setup.py", "sdist", "bdist_wheel")
log.Entry().Info("starting building python project:")
err := utils.RunExecutable(virutalEnvironmentPathMap["python"], flags...)
if err != nil {
return err
}
return nil
}
func publishWithTwine(config *pythonBuildOptions, utils pythonBuildUtils, installFlags []string) error {
installFlags = append(installFlags, "twine")
if err := utils.RunExecutable("python3", installFlags...); err != nil {
func createVirtualEnvironment(utils pythonBuildUtils, config *pythonBuildOptions, virutalEnvironmentPathMap map[string]string) error {
virtualEnvironmentFlags := []string{"-m", "venv", config.VirutalEnvironmentName}
err := utils.RunExecutable("python3", virtualEnvironmentFlags...)
if err != nil {
return err
}
if err := utils.RunExecutable("twine", "upload", "--username", config.TargetRepositoryUser,
err = utils.RunExecutable("bash", "-c", "source "+filepath.Join(config.VirutalEnvironmentName, "bin", "activate"))
if err != nil {
return err
}
virutalEnvironmentPathMap["pip"] = filepath.Join(config.VirutalEnvironmentName, "bin", "pip")
// venv will create symlinks to python3 inside the container
virutalEnvironmentPathMap["python"] = "python"
virutalEnvironmentPathMap["deactivate"] = filepath.Join(config.VirutalEnvironmentName, "bin", "deactivate")
return nil
}
func removeVirtualEnvironment(utils pythonBuildUtils, config *pythonBuildOptions) error {
err := utils.RemoveAll(config.VirutalEnvironmentName)
if err != nil {
return err
}
return nil
}
func runBOMCreationForPy(utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string, config *pythonBuildOptions) error {
pipInstallFlags = append(pipInstallFlags, "cyclonedx-bom")
if err := utils.RunExecutable(virutalEnvironmentPathMap["pip"], pipInstallFlags...); err != nil {
return err
}
virutalEnvironmentPathMap["cyclonedx"] = filepath.Join(config.VirutalEnvironmentName, "bin", "cyclonedx-bom")
if err := utils.RunExecutable(virutalEnvironmentPathMap["cyclonedx"], "--e", "--output", PyBomFilename); err != nil {
return err
}
return nil
}
func publishWithTwine(config *pythonBuildOptions, utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string) error {
pipInstallFlags = append(pipInstallFlags, "twine")
if err := utils.RunExecutable(virutalEnvironmentPathMap["pip"], pipInstallFlags...); err != nil {
return err
}
virutalEnvironmentPathMap["twine"] = filepath.Join(config.VirutalEnvironmentName, "bin", "twine")
if err := utils.RunExecutable(virutalEnvironmentPathMap["twine"], "upload", "--username", config.TargetRepositoryUser,
"--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL,
"dist/*"); err != nil {
return err

13
cmd/pythonBuild_generated.go
View File

@ -25,6 +25,7 @@ type pythonBuildOptions struct {
TargetRepositoryUser string `json:"targetRepositoryUser,omitempty"`
TargetRepositoryURL string `json:"targetRepositoryURL,omitempty"`
BuildSettingsInfo string `json:"buildSettingsInfo,omitempty"`
VirutalEnvironmentName string `json:"virutalEnvironmentName,omitempty"`
}
type pythonBuildCommonPipelineEnvironment struct {
@ -155,6 +156,7 @@ func addPythonBuildFlags(cmd *cobra.Command, stepConfig *pythonBuildOptions) {
cmd.Flags().StringVar(&stepConfig.TargetRepositoryUser, "targetRepositoryUser", os.Getenv("PIPER_targetRepositoryUser"), "Username for the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment.")
cmd.Flags().StringVar(&stepConfig.TargetRepositoryURL, "targetRepositoryURL", os.Getenv("PIPER_targetRepositoryURL"), "URL of the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment.")
cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the maven build . This information is typically used for compliance related processes.")
cmd.Flags().StringVar(&stepConfig.VirutalEnvironmentName, "virutalEnvironmentName", `piperBuild-env`, "name of the virtual environment that will be used for the build")
}
@ -252,10 +254,19 @@ func pythonBuildMetadata() config.StepData {
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_buildSettingsInfo"),
},
{
Name: "virutalEnvironmentName",
ResourceRef: []config.ResourceReference{},
Scope: []string{"STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: `piperBuild-env`,
},
},
},
Containers: []config.Container{
{Name: "python", Image: "python:3.9", Options: []config.Option{{Name: "-u", Value: "0"}}},
{Name: "python", Image: "python:3.9"},
},
Outputs: config.StepOutputs{
Resources: []config.StepResources{

76
cmd/pythonBuild_test.go
View File

@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"path/filepath"
"testing"
piperhttp "github.com/SAP/jenkins-library/pkg/http"
@ -42,20 +43,21 @@ func (f *pythonBuildMockUtils) GetConfig() *pythonBuildOptions {
func TestRunPythonBuild(t *testing.T) {
cpe := pythonBuildCommonPipelineEnvironment{}
t.Run("success - build", func(t *testing.T) {
config := pythonBuildOptions{}
config := pythonBuildOptions{
VirutalEnvironmentName: "dummy",
}
utils := newPythonBuildTestsUtils()
telemetryData := telemetry.CustomData{}
err := runPythonBuild(&config, &telemetryData, utils, &cpe)
assert.NoError(t, err)
runPythonBuild(&config, &telemetryData, utils, &cpe)
assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[0].Params)
assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params)
})
t.Run("failure - build failure", func(t *testing.T) {
config := pythonBuildOptions{}
utils := newPythonBuildTestsUtils()
utils.ShouldFailOnCommand = map[string]error{"python3 setup.py sdist bdist_wheel": fmt.Errorf("build failure")}
utils.ShouldFailOnCommand = map[string]error{"python setup.py sdist bdist_wheel": fmt.Errorf("build failure")}
telemetryData := telemetry.CustomData{}
err := runPythonBuild(&config, &telemetryData, utils, &cpe)
@ -68,60 +70,46 @@ func TestRunPythonBuild(t *testing.T) {
TargetRepositoryURL: "https://my.target.repository.local",
TargetRepositoryUser: "user",
TargetRepositoryPassword: "password",
VirutalEnvironmentName: "dummy",
}
utils := newPythonBuildTestsUtils()
telemetryData := telemetry.CustomData{}
err := runPythonBuild(&config, &telemetryData, utils, &cpe)
assert.NoError(t, err)
runPythonBuild(&config, &telemetryData, utils, &cpe)
assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[0].Params)
assert.Equal(t, "python3", utils.ExecMockRunner.Calls[1].Exec)
assert.Equal(t, []string{"-m", "pip", "install", "--upgrade", "twine"}, utils.ExecMockRunner.Calls[1].Params)
assert.Equal(t, "twine", utils.ExecMockRunner.Calls[2].Exec)
assert.Equal(t, []string{"-m", "venv", config.VirutalEnvironmentName}, utils.ExecMockRunner.Calls[0].Params)
assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec)
assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params)
assert.Equal(t, "python", utils.ExecMockRunner.Calls[2].Exec)
assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[2].Params)
assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec)
assert.Equal(t, []string{"install", "--upgrade", "twine"}, utils.ExecMockRunner.Calls[3].Params)
assert.Equal(t, filepath.Join("dummy", "bin", "twine"), utils.ExecMockRunner.Calls[4].Exec)
assert.Equal(t, []string{"upload", "--username", config.TargetRepositoryUser,
"--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL,
"dist/*"}, utils.ExecMockRunner.Calls[2].Params)
"dist/*"}, utils.ExecMockRunner.Calls[4].Params)
})
t.Run("success - create BOM", func(t *testing.T) {
config := pythonBuildOptions{
CreateBOM: true,
CreateBOM: true,
Publish: false,
VirutalEnvironmentName: "dummy",
}
utils := newPythonBuildTestsUtils()
telemetryData := telemetry.CustomData{}
err := runPythonBuild(&config, &telemetryData, utils, &cpe)
assert.NoError(t, err)
runPythonBuild(&config, &telemetryData, utils, &cpe)
// assert.NoError(t, err)
assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[0].Params)
assert.Equal(t, "python3", utils.ExecMockRunner.Calls[1].Exec)
assert.Equal(t, []string{"-m", "pip", "install", "--upgrade", "cyclonedx-bom"}, utils.ExecMockRunner.Calls[1].Params)
assert.Equal(t, "cyclonedx-bom", utils.ExecMockRunner.Calls[2].Exec)
assert.Equal(t, []string{"--e", "--output", "bom.xml"}, utils.ExecMockRunner.Calls[2].Params)
})
t.Run("failure - install pre-requisites for BOM creation", func(t *testing.T) {
config := pythonBuildOptions{
CreateBOM: true,
}
utils := newPythonBuildTestsUtils()
utils.ShouldFailOnCommand = map[string]error{"python3 -m pip install --upgrade cyclonedx-bom": fmt.Errorf("install failure")}
telemetryData := telemetry.CustomData{}
err := runPythonBuild(&config, &telemetryData, utils, &cpe)
assert.EqualError(t, err, "BOM creation failed: install failure")
})
t.Run("failure - install pre-requisites for Twine upload", func(t *testing.T) {
config := pythonBuildOptions{
Publish: true,
}
utils := newPythonBuildTestsUtils()
utils.ShouldFailOnCommand = map[string]error{"python3 -m pip install --upgrade twine": fmt.Errorf("install failure")}
telemetryData := telemetry.CustomData{}
err := runPythonBuild(&config, &telemetryData, utils, &cpe)
assert.EqualError(t, err, "failed to publish: install failure")
assert.Equal(t, []string{"-m", "venv", config.VirutalEnvironmentName}, utils.ExecMockRunner.Calls[0].Params)
assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec)
assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params)
assert.Equal(t, "python", utils.ExecMockRunner.Calls[2].Exec)
assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[2].Params)
assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec)
assert.Equal(t, []string{"install", "--upgrade", "cyclonedx-bom"}, utils.ExecMockRunner.Calls[3].Params)
assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-bom"), utils.ExecMockRunner.Calls[4].Exec)
assert.Equal(t, []string{"--e", "--output", "bom.xml"}, utils.ExecMockRunner.Calls[4].Params)
})
}

7
documentation/docs/steps/pythonBuild.md Normal file
View File

@ -0,0 +1,7 @@
# ${docGenStepName}
## ${docGenDescription}
## ${docGenParameters}
## ${docGenConfiguration}

1
documentation/mkdocs.yml
View File

@ -149,6 +149,7 @@ nav:
- piperPublishWarnings: steps/piperPublishWarnings.md
- prepareDefaultValues: steps/prepareDefaultValues.md
- protecodeExecuteScan: steps/protecodeExecuteScan.md
- pythonBuild: steps/pythonBuild.md
- seleniumExecuteTests: steps/seleniumExecuteTests.md
- setupCommonPipelineEnvironment: steps/setupCommonPipelineEnvironment.md
- shellExecute: steps/shellExecute.md

6
integration/integration_python_build_test.go
View File

@ -63,9 +63,9 @@ func TestBuildPythonProject(t *testing.T) {
}
output := string(content)
assert.Contains(t, output, "info pythonBuild - running command: python3 setup.py sdist bdist_wheel")
assert.Contains(t, output, "info pythonBuild - running command: python3 -m pip install --upgrade cyclonedx-bom")
assert.Contains(t, output, "info pythonBuild - running command: cyclonedx-bom --e --output bom.xml")
assert.Contains(t, output, "info pythonBuild - running command: python setup.py sdist bdist_wheel")
assert.Contains(t, output, "info pythonBuild - running command: piperBuild-env/bin/pip install --upgrade cyclonedx-bom")
assert.Contains(t, output, "info pythonBuild - running command: piperBuild-env/bin/cyclonedx-bom --e --output bom.xml")
assert.Contains(t, output, "info pythonBuild - SUCCESS")
//workaround to use test script util it is possible to set workdir for Exec call

13
resources/metadata/pythonBuild.yaml
View File

@ -28,6 +28,7 @@ spec:
- STEPS
- STAGES
- PARAMETERS
default: false
- name: targetRepositoryPassword
description: "Password for the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment."
type: string
@ -70,6 +71,14 @@ spec:
resourceRef:
- name: commonPipelineEnvironment
param: custom/buildSettingsInfo
- name: virutalEnvironmentName
type: string
description: name of the virtual environment that will be used for the build
scope:
- STEPS
- STAGES
- PARAMETERS
default: piperBuild-env
outputs:
resources:
- name: commonPipelineEnvironment
@ -79,7 +88,3 @@ spec:
containers:
- name: python
image: python:3.9
# workingDir: /home/node
options:
- name: -u
value: "0"