From 0b8b6f2b0cfbef6f0ed93c845c1459c92752cb15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 31 Mar 2020 15:16:18 +0200 Subject: [PATCH] Migrate stage artifact deployment from Cloud SDK Pipeline-Lib (#1324) Co-authored-by: Daniel Kurzynski Co-authored-by: Florian Geckeler --- cmd/nexusUpload.go | 28 ++++----- cmd/nexusUpload_generated.go | 16 ++--- cmd/nexusUpload_test.go | 34 ++-------- pkg/nexus/nexus.go | 56 +++++++++-------- pkg/nexus/nexus_test.go | 8 ++- resources/metadata/nexusUpload.yaml | 20 +++--- test/groovy/CommonStepsTest.groovy | 1 + vars/nexusUpload.groovy | 11 ++++ vars/piperExecuteBin.groovy | 1 + ...iperPipelineStageArtifactDeployment.groovy | 63 +++++++++++++++++++ 10 files changed, 149 insertions(+), 89 deletions(-) create mode 100644 vars/piperPipelineStageArtifactDeployment.groovy diff --git a/cmd/nexusUpload.go b/cmd/nexusUpload.go index 9fe0d8c44..faa16a9b0 100644 --- a/cmd/nexusUpload.go +++ b/cmd/nexusUpload.go @@ -135,18 +135,12 @@ func uploadMTA(utils nexusUploadUtils, uploader nexus.Uploader, options *nexusUp // This will fail anyway if the file doesn't exist mtaPath = "mta.yml" } - version, err := getVersionFromMtaFile(utils, mtaPath) - var artifactID = options.ArtifactID - if artifactID == "" { - artifactID = utils.getEnvParameter(".pipeline/commonPipelineEnvironment/configuration", "artifactId") - if artifactID == "" { - err = fmt.Errorf("the 'artifactId' parameter was not provided and could not be retrieved from the Common Pipeline Environment") - } else { - log.Entry().Debugf("mtar artifact id from CPE: '%s'", artifactID) - } - } + mtaInfo, err := getInfoFromMtaFile(utils, mtaPath) if err == nil { - err = uploader.SetInfo(options.GroupID, artifactID, version) + if options.ArtifactID != "" { + mtaInfo.ID = options.ArtifactID + } + err = uploader.SetInfo(options.GroupID, mtaInfo.ID, mtaInfo.Version) if err == nexus.ErrEmptyVersion { err = fmt.Errorf("the project descriptor file 'mta.yaml' has an invalid version: %w", err) } @@ -170,25 +164,25 @@ type mtaYaml struct { Version string `json:"version"` } -func getVersionFromMtaFile(utils nexusUploadUtils, filePath string) (string, error) { +func getInfoFromMtaFile(utils nexusUploadUtils, filePath string) (*mtaYaml, error) { mtaYamlContent, err := utils.fileRead(filePath) if err != nil { - return "", fmt.Errorf("could not read from required project descriptor file '%s'", + return nil, fmt.Errorf("could not read from required project descriptor file '%s'", filePath) } - return getVersionFromMtaYaml(mtaYamlContent, filePath) + return getInfoFromMtaYaml(mtaYamlContent, filePath) } -func getVersionFromMtaYaml(mtaYamlContent []byte, filePath string) (string, error) { +func getInfoFromMtaYaml(mtaYamlContent []byte, filePath string) (*mtaYaml, error) { var mtaYaml mtaYaml err := yaml.Unmarshal(mtaYamlContent, &mtaYaml) if err != nil { // Eat the original error as it is unhelpful and confusingly mentions JSON, while the // user thinks it should parse YAML (it is transposed by the implementation). - return "", fmt.Errorf("failed to parse contents of the project descriptor file '%s'", + return nil, fmt.Errorf("failed to parse contents of the project descriptor file '%s'", filePath) } - return mtaYaml.Version, nil + return &mtaYaml, nil } func createMavenExecuteOptions(options *nexusUploadOptions) maven.ExecuteOptions { diff --git a/cmd/nexusUpload_generated.go b/cmd/nexusUpload_generated.go index 753a14b02..bbcb95ff0 100644 --- a/cmd/nexusUpload_generated.go +++ b/cmd/nexusUpload_generated.go @@ -93,7 +93,7 @@ func nexusUploadMetadata() config.StepData { Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: false, - Aliases: []config.Alias{}, + Aliases: []config.Alias{{Name: "nexus/version"}}, }, { Name: "url", @@ -101,7 +101,7 @@ func nexusUploadMetadata() config.StepData { Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: true, - Aliases: []config.Alias{}, + Aliases: []config.Alias{{Name: "nexus/url"}}, }, { Name: "repository", @@ -109,7 +109,7 @@ func nexusUploadMetadata() config.StepData { Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: true, - Aliases: []config.Alias{}, + Aliases: []config.Alias{{Name: "nexus/repository"}}, }, { Name: "groupId", @@ -117,12 +117,12 @@ func nexusUploadMetadata() config.StepData { Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: false, - Aliases: []config.Alias{}, + Aliases: []config.Alias{{Name: "nexus/groupId"}}, }, { Name: "artifactId", ResourceRef: []config.ResourceReference{}, - Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Scope: []string{"PARAMETERS"}, Type: "string", Mandatory: false, Aliases: []config.Alias{}, @@ -149,12 +149,12 @@ func nexusUploadMetadata() config.StepData { Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: false, - Aliases: []config.Alias{}, + Aliases: []config.Alias{{Name: "nexus/additionalClassifiers"}}, }, { Name: "user", ResourceRef: []config.ResourceReference{}, - Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Scope: []string{"PARAMETERS"}, Type: "string", Mandatory: false, Aliases: []config.Alias{}, @@ -162,7 +162,7 @@ func nexusUploadMetadata() config.StepData { { Name: "password", ResourceRef: []config.ResourceReference{}, - Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Scope: []string{"PARAMETERS"}, Type: "string", Mandatory: false, Aliases: []config.Alias{}, diff --git a/cmd/nexusUpload_test.go b/cmd/nexusUpload_test.go index 16b60c5e0..a133fca94 100644 --- a/cmd/nexusUpload_test.go +++ b/cmd/nexusUpload_test.go @@ -166,18 +166,20 @@ func TestUploadMTAProjects(t *testing.T) { assert.Equal(t, 0, len(uploader.GetArtifacts())) assert.Equal(t, 0, len(uploader.uploadedArtifacts)) }) - t.Run("Uploading MTA project without artifactId parameter fails", func(t *testing.T) { + t.Run("Uploading MTA project without artifactId parameter works", func(t *testing.T) { utils := newMockUtilsBundle(true, false) utils.files["mta.yaml"] = testMtaYml + utils.files["test.mtar"] = []byte("contentsOfMtar") utils.cpe[".pipeline/commonPipelineEnvironment/mtarFilePath"] = "test.mtar" uploader := mockUploader{} options := createOptions() options.ArtifactID = "" err := runNexusUpload(&utils, &uploader, &options) - assert.EqualError(t, err, "the 'artifactId' parameter was not provided and could not be retrieved from the Common Pipeline Environment") - assert.Equal(t, 0, len(uploader.GetArtifacts())) - assert.Equal(t, 0, len(uploader.uploadedArtifacts)) + if assert.NoError(t, err) { + assert.Equal(t, 2, len(uploader.uploadedArtifacts)) + assert.Equal(t, "test", uploader.GetArtifactsID()) + } }) t.Run("Uploading MTA project fails due to missing yaml file", func(t *testing.T) { utils := newMockUtilsBundle(true, false) @@ -274,30 +276,6 @@ func TestUploadMTAProjects(t *testing.T) { assert.Equal(t, "0.3.0", uploader.GetArtifactsVersion()) assert.Equal(t, "artifact.id", uploader.GetArtifactsID()) - artifacts := uploader.uploadedArtifacts - if assert.Equal(t, 2, len(artifacts)) { - assert.Equal(t, "mta.yml", artifacts[0].File) - assert.Equal(t, "yaml", artifacts[0].Type) - - assert.Equal(t, "test.mtar", artifacts[1].File) - assert.Equal(t, "mtar", artifacts[1].Type) - } - }) - t.Run("Test uploading mta.yml project works with artifactID from CPE", func(t *testing.T) { - utils := newMockUtilsBundle(true, false) - utils.files["mta.yml"] = testMtaYml - utils.files["test.mtar"] = []byte("contentsOfMtar") - utils.cpe[".pipeline/commonPipelineEnvironment/mtarFilePath"] = "test.mtar" - utils.cpe[".pipeline/commonPipelineEnvironment/configuration/artifactId"] = "my-artifact-id" - uploader := mockUploader{} - options := createOptions() - // Clear artifact ID to trigger reading it from the CPE - options.ArtifactID = "" - - err := runNexusUpload(&utils, &uploader, &options) - assert.NoError(t, err, "expected mta.yml project upload to work") - assert.Equal(t, "my-artifact-id", uploader.GetArtifactsID()) - artifacts := uploader.uploadedArtifacts if assert.Equal(t, 2, len(artifacts)) { assert.Equal(t, "mta.yml", artifacts[0].File) diff --git a/pkg/nexus/nexus.go b/pkg/nexus/nexus.go index 37bf083e0..65e94b1ca 100644 --- a/pkg/nexus/nexus.go +++ b/pkg/nexus/nexus.go @@ -48,6 +48,36 @@ func (nexusUpload *Upload) SetRepoURL(nexusURL, nexusVersion, repository string) return nil } +func getBaseURL(nexusURL, nexusVersion, repository string) (string, error) { + if nexusURL == "" { + return "", errors.New("nexusURL must not be empty") + } + nexusURL = strings.ToLower(nexusURL) + var protocols = []string{"http://", "https://"} + for _, protocol := range protocols { + if strings.HasPrefix(nexusURL, protocol) { + nexusURL = strings.TrimPrefix(nexusURL, protocol) + break + } + } + if repository == "" { + return "", errors.New("repository must not be empty") + } + baseURL := nexusURL + switch nexusVersion { + case "nexus2": + baseURL += "/content/repositories/" + case "nexus3": + baseURL += "/repository/" + default: + return "", fmt.Errorf("unsupported Nexus version '%s', must be 'nexus2' or 'nexus3'", nexusVersion) + } + baseURL += repository + "/" + // Replace any double slashes, as nexus does not like them + baseURL = strings.ReplaceAll(baseURL, "//", "/") + return baseURL, nil +} + // GetRepoURL returns the base URL for the nexus repository. func (nexusUpload *Upload) GetRepoURL() string { return nexusUpload.repoURL @@ -144,29 +174,3 @@ func (nexusUpload *Upload) GetArtifacts() []ArtifactDescription { func (nexusUpload *Upload) Clear() { nexusUpload.artifacts = []ArtifactDescription{} } - -func getBaseURL(nexusURL, nexusVersion, repository string) (string, error) { - if nexusURL == "" { - return "", errors.New("nexusURL must not be empty") - } - nexusURL = strings.ToLower(nexusURL) - if strings.HasPrefix(nexusURL, "http://") || strings.HasPrefix(nexusURL, "https://") { - return "", errors.New("nexusURL must not start with 'http://' or 'https://'") - } - if repository == "" { - return "", errors.New("repository must not be empty") - } - baseURL := nexusURL - switch nexusVersion { - case "nexus2": - baseURL += "/content/repositories/" - case "nexus3": - baseURL += "/repository/" - default: - return "", fmt.Errorf("unsupported Nexus version '%s', must be 'nexus2' or 'nexus3'", nexusVersion) - } - baseURL += repository + "/" - // Replace any double slashes, as nexus does not like them - baseURL = strings.ReplaceAll(baseURL, "//", "/") - return baseURL, nil -} diff --git a/pkg/nexus/nexus_test.go b/pkg/nexus/nexus_test.go index f3a3db300..518565fff 100644 --- a/pkg/nexus/nexus_test.go +++ b/pkg/nexus/nexus_test.go @@ -110,12 +110,16 @@ func TestSetBaseURL(t *testing.T) { t.Run("Test host wrongly includes protocol http://", func(t *testing.T) { nexusUpload := Upload{} err := nexusUpload.SetRepoURL("htTp://localhost:8081", "nexus3", "maven-releases") - assert.Error(t, err, "Expected SetRepoURL() to fail (invalid host)") + if assert.NoError(t, err, "Expected SetRepoURL() to work") { + assert.Equal(t, "localhost:8081/repository/maven-releases/", nexusUpload.repoURL) + } }) t.Run("Test host wrongly includes protocol https://", func(t *testing.T) { nexusUpload := Upload{} err := nexusUpload.SetRepoURL("htTpS://localhost:8081", "nexus3", "maven-releases") - assert.Error(t, err, "Expected SetRepoURL() to fail (invalid host)") + if assert.NoError(t, err, "Expected SetRepoURL() to work") { + assert.Equal(t, "localhost:8081/repository/maven-releases/", nexusUpload.repoURL) + } }) t.Run("Test invalid version provided", func(t *testing.T) { nexusUpload := Upload{} diff --git a/resources/metadata/nexusUpload.yaml b/resources/metadata/nexusUpload.yaml index 100552636..0594ae00e 100644 --- a/resources/metadata/nexusUpload.yaml +++ b/resources/metadata/nexusUpload.yaml @@ -12,6 +12,8 @@ spec: - name: nexusCredentialsId description: The technical username/password credential for accessing the nexus endpoint. type: jenkins + aliases: + - name: nexus/credentialsId params: - name: version type: string @@ -22,6 +24,8 @@ spec: - STEPS mandatory: false default: nexus3 + aliases: + - name: nexus/version - name: url type: string description: URL of the nexus. The scheme part of the URL will not be considered, because only http is supported. @@ -30,6 +34,8 @@ spec: - STAGES - STEPS mandatory: true + aliases: + - name: nexus/url - name: repository type: string description: Name of the nexus repository. @@ -38,7 +44,8 @@ spec: - STAGES - STEPS mandatory: true - default: + aliases: + - name: nexus/repository - name: groupId type: string description: Group ID of the artifacts. Only used in MTA projects, ignored for Maven. @@ -46,14 +53,13 @@ spec: - PARAMETERS - STAGES - STEPS - mandatory: false + aliases: + - name: nexus/groupId - name: artifactId type: string description: The artifact ID used for both the .mtar and mta.yaml files deployed for MTA projects, ignored for Maven. scope: - PARAMETERS - - STAGES - - STEPS - name: globalSettingsFile type: string description: Path to the mvn settings file that should be used as global settings file. @@ -81,20 +87,18 @@ spec: - PARAMETERS - STAGES - STEPS + aliases: + - name: nexus/additionalClassifiers - name: user type: string description: User scope: - PARAMETERS - - STAGES - - STEPS - name: password type: string description: Password scope: - PARAMETERS - - STAGES - - STEPS containers: - name: mvn image: maven:3.6-jdk-8 diff --git a/test/groovy/CommonStepsTest.groovy b/test/groovy/CommonStepsTest.groovy index 97c3cd2ec..ff4d94d12 100644 --- a/test/groovy/CommonStepsTest.groovy +++ b/test/groovy/CommonStepsTest.groovy @@ -133,6 +133,7 @@ public class CommonStepsTest extends BasePiperTest{ 'nexusUpload', //implementing new golang pattern without fields 'mavenBuild', //implementing new golang pattern without fields 'mavenExecuteStaticCodeChecks', //implementing new golang pattern without fields + 'piperPipelineStageArtifactDeployment', //stage without step flags ] @Test diff --git a/vars/nexusUpload.groovy b/vars/nexusUpload.groovy index 8bdb38801..fd253daf6 100644 --- a/vars/nexusUpload.groovy +++ b/vars/nexusUpload.groovy @@ -1,11 +1,22 @@ +import com.sap.piper.DownloadCacheUtils import groovy.transform.Field +import static groovy.json.JsonOutput.toJson + @Field String STEP_NAME = getClass().getName() @Field String METADATA_FILE = 'metadata/nexusUpload.yaml' //Metadata maintained in file project://resources/metadata/nexusUpload.yaml void call(Map parameters = [:]) { + // Replace 'additionalClassifiers' List with JSON encoded String. + // This is currently necessary, since the go code doesn't support complex/arbitrary parameter types. + // TODO: Support complex/structured types of parameters in piper-go + if (parameters.additionalClassifiers) { + parameters.additionalClassifiers = "${toJson(parameters.additionalClassifiers as List)}" + } + parameters = DownloadCacheUtils.injectDownloadCacheInMavenParameters(parameters.script, parameters) + List credentials = [[type: 'usernamePassword', id: 'nexusCredentialsId', env: ['PIPER_username', 'PIPER_password']]] piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials) } diff --git a/vars/piperExecuteBin.groovy b/vars/piperExecuteBin.groovy index 62f2b365d..aa480388a 100644 --- a/vars/piperExecuteBin.groovy +++ b/vars/piperExecuteBin.groovy @@ -68,6 +68,7 @@ static String getCustomDefaultConfigsArg() { static String getCustomConfigArg(def script) { if (script?.commonPipelineEnvironment?.configurationFile + && script.commonPipelineEnvironment.configurationFile != '.pipeline/config.yml' && script.commonPipelineEnvironment.configurationFile != '.pipeline/config.yaml') { return " --customConfig ${BashUtils.quoteAndEscape(script.commonPipelineEnvironment.configurationFile)}" } diff --git a/vars/piperPipelineStageArtifactDeployment.groovy b/vars/piperPipelineStageArtifactDeployment.groovy new file mode 100644 index 000000000..c61fbf50b --- /dev/null +++ b/vars/piperPipelineStageArtifactDeployment.groovy @@ -0,0 +1,63 @@ +import com.sap.piper.ConfigurationHelper +import com.sap.piper.GenerateStageDocumentation +import com.sap.piper.ReportAggregator +import groovy.transform.Field + +import static com.sap.piper.Prerequisites.checkScript + +@Field String STEP_NAME = getClass().getName() + +@Field Set GENERAL_CONFIG_KEYS = [] +@Field Set STAGE_STEP_KEYS = [] +@Field Set STAGE_CONFIG_KEYS = STAGE_STEP_KEYS.plus([ + /** Parameters for deployment to a Nexus Repository Manager. */ + 'nexus' +]) +@Field Set PARAMETER_KEYS = STAGE_CONFIG_KEYS + +/** + * This stage is responsible for releasing/deploying artifacts to a Nexus Repository Manager.
+ */ +@GenerateStageDocumentation(defaultStageName = 'artifactDeployment') +void call(Map parameters = [:]) { + String stageName = 'artifactDeployment' + final script = checkScript(this, parameters) ?: this + + Map config = ConfigurationHelper.newInstance(this) + .loadStepDefaults() + .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS) + .mixinStageConfig(script.commonPipelineEnvironment, stageName, STAGE_CONFIG_KEYS) + .mixin(parameters, PARAMETER_KEYS) + .withMandatoryProperty('nexus') + .use() + + piperStageWrapper(stageName: stageName, script: script) { + + def commonPipelineEnvironment = script.commonPipelineEnvironment + List unstableSteps = commonPipelineEnvironment?.getValue('unstableSteps') ?: [] + if (unstableSteps) { + piperPipelineStageConfirm script: script + unstableSteps = [] + commonPipelineEnvironment.setValue('unstableSteps', unstableSteps) + } + + Map nexusConfig = config.nexus as Map + + // Pull additionalClassifiers param from resolved config here for legacy compatibility. + // The parameter will become obsolete soon. + Map nexusUploadParams = [ + script: script, + additionalClassifiers: nexusConfig.additionalClassifiers, + ] + + if (nexusConfig.credentialsId) { + nexusUploadParams.nexusCredentialsId = nexusConfig.credentialsId + } + + withEnv(["STAGE_NAME=${stageName}"]) { + nexusUpload(nexusUploadParams) + } + + ReportAggregator.instance.reportDeploymentToNexus() + } +}