From aefe9243e034bc297aa5a1819ae9364a1a8a2d49 Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Fri, 13 Dec 2019 16:05:55 +0100 Subject: [PATCH 01/10] xsDeploy with go --- test/groovy/CommonStepsTest.groovy | 3 +- test/groovy/XsDeployTest.groovy | 463 ++++++----------------------- vars/xsDeploy.groovy | 361 ++++++---------------- 3 files changed, 195 insertions(+), 632 deletions(-) diff --git a/test/groovy/CommonStepsTest.groovy b/test/groovy/CommonStepsTest.groovy index d8ebf3a0e..69cce2c05 100644 --- a/test/groovy/CommonStepsTest.groovy +++ b/test/groovy/CommonStepsTest.groovy @@ -115,7 +115,8 @@ public class CommonStepsTest extends BasePiperTest{ 'handlePipelineStepErrors', // special step (infrastructure) 'piperStageWrapper', //intended to be called from within stages 'buildSetResult', - 'githubPublishRelease' //implementing new golang pattern without fields + 'githubPublishRelease', //implementing new golang pattern without fields + 'xsDeploy', //implementing new golang pattern without fields ] @Test diff --git a/test/groovy/XsDeployTest.groovy b/test/groovy/XsDeployTest.groovy index ff5fdf608..a260bdc98 100644 --- a/test/groovy/XsDeployTest.groovy +++ b/test/groovy/XsDeployTest.groovy @@ -1,252 +1,105 @@ -import static org.junit.Assert.assertThat - -import org.hamcrest.Matchers - import static org.hamcrest.Matchers.allOf import static org.hamcrest.Matchers.contains import static org.hamcrest.Matchers.containsString -import static org.hamcrest.Matchers.hasSize +import static org.hamcrest.Matchers.equalTo import static org.hamcrest.Matchers.is +import static org.hamcrest.Matchers.not +import static org.hamcrest.Matchers.nullValue +import static org.junit.Assert.assertThat +import org.hamcrest.Matchers +import org.hamcrest.core.IsNull +import org.junit.Before import org.junit.Rule import org.junit.Test import org.junit.rules.ExpectedException import org.junit.rules.RuleChain +import com.sap.piper.PiperGoUtils + +import hudson.AbortException import util.BasePiperTest import util.CommandLineMatcher import util.JenkinsCredentialsRule import util.JenkinsDockerExecuteRule -import util.JenkinsFileExistsRule import util.JenkinsLockRule -import util.JenkinsLoggingRule +import util.JenkinsReadJsonRule import util.JenkinsReadYamlRule import util.JenkinsShellCallRule import util.JenkinsStepRule +import util.JenkinsWriteFileRule import util.Rules -import com.sap.piper.JenkinsUtils - -import hudson.AbortException - class XsDeployTest extends BasePiperTest { private ExpectedException thrown = ExpectedException.none() - private List existingFiles = [ - '.xsconfig', - 'myApp.mta' - ] - private JenkinsStepRule stepRule = new JenkinsStepRule(this) private JenkinsShellCallRule shellRule = new JenkinsShellCallRule(this) private JenkinsLockRule lockRule = new JenkinsLockRule(this) - private JenkinsLoggingRule logRule = new JenkinsLoggingRule(this) + private JenkinsDockerExecuteRule dockerRule = new JenkinsDockerExecuteRule(this) + private JenkinsWriteFileRule writeFileRule = new JenkinsWriteFileRule(this) + + List env @Rule public RuleChain ruleChain = Rules.getCommonRules(this) .around(new JenkinsReadYamlRule(this)) + .around(new JenkinsReadJsonRule(this)) .around(stepRule) - .around(new JenkinsDockerExecuteRule(this)) + .around(dockerRule) + .around(writeFileRule) .around(new JenkinsCredentialsRule(this) .withCredentials('myCreds', 'cred_xs', 'topSecret')) - .around(new JenkinsFileExistsRule(this, existingFiles)) .around(lockRule) .around(shellRule) - .around(logRule) .around(thrown) - @Test - public void testSanityChecks() { - - thrown.expect(IllegalArgumentException) - thrown.expectMessage( - allOf( - containsString('ERROR - NO VALUE AVAILABLE FOR:'), - containsString('apiUrl'), - containsString('org'), - containsString('space'), - containsString('mtaPath'))) - - stepRule.step.xsDeploy(script: nullScript) + private PiperGoUtils goUtils = new PiperGoUtils(null) { + void unstashPiperBin() { + } } - @Test - public void testLoginFailed() { - - thrown.expect(AbortException) - thrown.expectMessage('xs login failed') - - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '#!/bin/bash xs login .*', 1) - - try { - stepRule.step.xsDeploy( - script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - mtaPath: 'myApp.mta' - ) - } catch(AbortException e ) { - - assertThat(shellRule.shell, - allOf( - // first item: the login attempt - // second item: we try to provide the logs - hasSize(2), - new CommandLineMatcher() - .hasProlog("#!/bin/bash") - .hasSnippet('xs login'), - new CommandLineMatcher() - .hasProlog('LOG_FOLDER') - .hasSnippet('cat \\$\\{LOG_FOLDER\\}/\\*') - ) - ) - throw e - } + @Before + public void init() { + helper.registerAllowedMethod('withEnv', [List, Closure], {l, c -> env = l; c()}) + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --contextConfig --stepMetadata.*', '{"dockerImage": "xs", "credentialsId":"myCreds"}') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --stepMetadata.*', '{"mode": "BG_DEPLOY", "action": "NONE", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + nullScript.commonPipelineEnvironment.xsDeploymentId = null } @Test public void testDeployFailed() { thrown.expect(AbortException) - thrown.expectMessage('Failed command(s): [xs deploy]. Check earlier log for details.') + thrown.expectMessage('script returned exit code 1') - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '#!/bin/bash.*xs deploy .*', {throw new AbortException()}) - - try { - stepRule.step.xsDeploy( - script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - mtaPath: 'myApp.mta' - ) - } catch(AbortException e ) { - - assertThat(shellRule.shell, - allOf( - hasSize(4), - new CommandLineMatcher() - .hasProlog("#!/bin/bash") - .hasSnippet('xs login'), - new CommandLineMatcher() - .hasProlog("#!/bin/bash") - .hasSnippet('xs deploy'), - new CommandLineMatcher() - .hasProlog('#!/bin/bash') - .hasSnippet('xs logout'), // logout must be present in case deployment failed. - new CommandLineMatcher() - .hasProlog('') - .hasSnippet('rm \\$\\{XSCONFIG\\}') // remove the session file after logout - ) - ) - throw e - } - } - - @Test - public void testNothingHappensWhenModeIsNone() { + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*xsDeploy .*', { throw new AbortException('script returned exit code 1')}) stepRule.step.xsDeploy( script: nullScript, - mode: 'NONE' + piperGoUtils: goUtils, ) - - assertThat(logRule.log, containsString('Deployment skipped intentionally.')) - assertThat(shellRule.shell, hasSize(0)) } @Test - public void testDeploymentFailsWhenDeployableIsNotPresent() { - - thrown.expect(AbortException) - thrown.expectMessage('Deployable \'myApp.mta\' does not exist.') - - existingFiles.remove('myApp.mta') - - try { - stepRule.step.xsDeploy( - script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - mtaPath: 'myApp.mta' - ) - } catch(AbortException e) { - - // no shell operation happened in this case. - assertThat(shellRule.shell.size(), is(0)) - - throw e - } - } - - @Test - public void testDeployStraighForward() { - - stepRule.step.xsDeploy( - script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - deployOpts: '-t 60', - mtaPath: 'myApp.mta' - ) - - assertThat(shellRule.shell, - allOf( - new CommandLineMatcher() - .hasProlog("#!/bin/bash xs login") - .hasSnippet('xs login') - .hasOption('a', 'https://example.org/xs') - .hasOption('u', 'cred_xs') - .hasSingleQuotedOption('p', 'topSecret') - .hasOption('o', 'myOrg') - .hasOption('s', 'mySpace'), - new CommandLineMatcher() - .hasProlog("#!/bin/bash") - .hasSnippet('xs deploy') - .hasOption('t', '60') - .hasArgument('\'myApp.mta\''), - new CommandLineMatcher() - .hasProlog("#!/bin/bash") - .hasSnippet('xs logout') - ) - ) - - assertThat(lockRule.getLockResources(), contains('xsDeploy:https://example.org/xs:myOrg:mySpace')) - - } - - @Test - public void testInvalidDeploymentModeProviced() { + public void testInvalidDeploymentModeProvided() { thrown.expect(IllegalArgumentException) thrown.expectMessage('No enum constant') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --stepMetadata.*', '{"mode": "DOES_NOT_EXIST", "action": "NONE", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + stepRule.step.xsDeploy( script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - deployOpts: '-t 60', - mtaPath: 'myApp.mta', - mode: 'DOES_NOT_EXIST' + piperGoUtils: goUtils, ) } @Test - public void testActionProvidedForStandardDeployment() { + public void testParametersViaSignature() { - thrown.expect(AbortException) - thrown.expectMessage( - 'Cannot perform action \'resume\' in mode \'deploy\'. Only action \'none\' is allowed.') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*xsDeploy .*', '{"operationId": "1234"}') stepRule.step.xsDeploy( script: nullScript, @@ -256,109 +109,88 @@ class XsDeployTest extends BasePiperTest { credentialsId: 'myCreds', deployOpts: '-t 60', mtaPath: 'myApp.mta', - mode: 'DEPLOY', // this is the default anyway - action: 'RESUME' + mode: 'DEPLOY', + action: 'NONE', + piperGoUtils: goUtils ) + + + // nota bene: script and piperGoUtils are not contained in the json below. + assertThat(env*.toString(), contains('PIPER_parametersJSON={"apiUrl":"https://example.org/xs","org":"myOrg","space":"mySpace","credentialsId":"myCreds","deployOpts":"-t 60","mtaPath":"myApp.mta","mode":"DEPLOY","action":"NONE"}')) } @Test - public void testBlueGreenDeployFailes() { + public void testBlueGreenDeployInit() { - thrown.expect(AbortException) - thrown.expectMessage('Failed command(s): [xs bg-deploy]') + // + // Only difference between bg deploy and standard deploy is in the config. + // The surrounding behavior is the same. Hence there is no dedicated test here + // in the groovy layer for standard deploy + // - logRule.expect('Something went wrong') + boolean unstashCalled - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '#!/bin/bash.*xs bg-deploy .*', - { throw new AbortException('Something went wrong.') }) + assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, nullValue()) + + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*xsDeploy .*', '{"operationId": "1234"}') - try { - stepRule.step.xsDeploy( - script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - mtaPath: 'myApp.mta', - mode: 'BG_DEPLOY' - ) - } catch(AbortException e) { - - // in case there is a deployment failure we have to logout also for bg-deployments - assertThat(shellRule.shell, - new CommandLineMatcher() - .hasProlog('#!/bin/bash') - .hasSnippet('xs logout') - ) - - throw e + goUtils = new PiperGoUtils(null) { + void unstashPiperBin() { + unstashCalled = true + } } -} - - @Test - public void testBlueGreenDeployStraighForward() { - - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '#!/bin/bash.*xs bg-deploy .*', - ((CharSequence)''' | - | - |Uploading 1 files: - |/myFolder/my.mtar - |File upload finished - | - |Detected MTA schema version: "3.1.0" - |Detected deploy target as "myOrg mySpace" - |Detected deployed MTA with ID "my_mta" and version "0.0.1" - |Deployed MTA color: blue - |New MTA color: green - |Detected new MTA version: "0.0.1" - |Deployed MTA version: 0.0.1 - |Service "xxx" is not modified and will not be updated - |Creating application "db-green" from MTA module "xx"... - |Uploading application "xx-green"... - |Staging application "xx-green"... - |Application "xx-green" staged - |Executing task "deploy" on application "xx-green"... - |Task execution status: succeeded - |Process has entered validation phase. After testing your new deployment you can resume or abort the process. - |Use "xs bg-deploy -i 1234 -a resume" to resume the process. - |Use "xs bg-deploy -i 1234 -a abort" to abort the process. - |Hint: Use the '--no-confirm' option of the bg-deploy command to skip this phase. - |''').stripMargin()) - stepRule.step.xsDeploy( script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - deployOpts: '-t 60', - mtaPath: 'myApp.mta', - mode: 'BG_DEPLOY' + piperGoUtils: goUtils ) + assertThat(unstashCalled, equalTo(true)) + assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, is('1234')) + assertThat(writeFileRule.files.keySet(), contains('metadata/xsDeploy.yaml')) + + assertThat(dockerRule.dockerParams.dockerImage, equalTo('xs')) + assertThat(dockerRule.dockerParams.dockerPullImage, equalTo(false)) + assertThat(shellRule.shell, allOf( new CommandLineMatcher() - .hasProlog("#!/bin/bash xs login") - .hasOption('a', 'https://example.org/xs') - .hasOption('u', 'cred_xs') - .hasSingleQuotedOption('p', 'topSecret') - .hasOption('o', 'myOrg') - .hasOption('s', 'mySpace'), + .hasProlog('./piper version'), new CommandLineMatcher() - .hasProlog("#!/bin/bash") - .hasOption('t', '60') - .hasArgument('\'myApp.mta\''), + .hasProlog('./piper getConfig --contextConfig --stepMetadata \'metadata/xsDeploy.yaml\''), new CommandLineMatcher() - .hasProlog("#!/bin/bash") + .hasProlog('./piper getConfig --stepMetadata \'metadata/xsDeploy.yaml\''), + new CommandLineMatcher() + .hasProlog('#!/bin/bash ./piper xsDeploy --user \\$\\{USERNAME\\} --password \\$\\{PASSWORD\\}'), + not(new CommandLineMatcher() + .hasProlog('#!/bin/bash ./piper xsDeploy --user \\$\\{USERNAME\\} --password \\$\\{PASSWORD\\} --operationId')) ) ) assertThat(lockRule.getLockResources(), contains('xsDeploy:https://example.org/xs:myOrg:mySpace')) } + @Test + public void testBlueGreenDeployResume() { + + nullScript.commonPipelineEnvironment.xsDeploymentId = '1234' + + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --stepMetadata.*', '{"mode": "BG_DEPLOY", "action": "RESUME", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + + stepRule.step.xsDeploy( + script: nullScript, + piperGoUtils: goUtils + ) + + assertThat(shellRule.shell, + new CommandLineMatcher() + .hasProlog('#!/bin/bash ./piper xsDeploy --user \\$\\{USERNAME\\} --password \\$\\{PASSWORD\\} --operationId 1234') + ) + + assertThat(lockRule.getLockResources(), contains('xsDeploy:https://example.org/xs:myOrg:mySpace')) + } + @Test public void testBlueGreenDeployResumeWithoutDeploymentId() { @@ -367,114 +199,17 @@ class XsDeployTest extends BasePiperTest { thrown.expect(IllegalArgumentException) thrown.expectMessage( allOf( - containsString('No deployment id provided'), + containsString('No operationId provided'), containsString('Was there a deployment before?'))) - nullScript.commonPipelineEnvironment.xsDeploymentId = null // is null anyway, just for clarification + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --stepMetadata.*', '{"mode": "BG_DEPLOY", "action": "RESUME", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + + assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, nullValue()) stepRule.step.xsDeploy( script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - mode: 'BG_DEPLOY', - action: 'RESUME' + piperGoUtils: goUtils, + failOnError: true, ) } - - @Test - public void testBlueGreenDeployWithoutExistingSession() { - - thrown.expect(AbortException) - thrown.expectMessage( - 'For the current configuration an already existing session is required.' + - ' But there is no already existing session') - - existingFiles.remove('.xsconfig') - - stepRule.step.xsDeploy( - script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - mode: 'BG_DEPLOY', - action: 'RESUME' - ) - - } - - @Test - public void testBlueGreenDeployResumeFails() { - - // e.g. we try to resume a deployment which did not succeed or which was already resumed or aborted. - - thrown.expect(AbortException) - thrown.expectMessage('Failed command(s): [xs bg-deploy -a resume].') - - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'xs bg-deploy -i .*', 1) - - nullScript.commonPipelineEnvironment.xsDeploymentId = '1234' - - try { - stepRule.step.xsDeploy( - script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - mode: 'BG_DEPLOY', - action: 'RESUME' - ) - } catch(AbortException e) { - - // logout must happen also in case of a failed deployment - assertThat(shellRule.shell, - new CommandLineMatcher() - .hasProlog('') - .hasSnippet('xs logout')) - throw e - } - } - - @Test - public void testBlueGreenDeployResume() { - - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'xs bg-deploy -i .*', 0) - - nullScript.commonPipelineEnvironment.xsDeploymentId = '1234' - - stepRule.step.xsDeploy( - script: nullScript, - apiUrl: 'https://example.org/xs', - org: 'myOrg', - space: 'mySpace', - credentialsId: 'myCreds', - mode: 'BG_DEPLOY', - action: 'RESUME' - ) - - // there is no login in case of a resume since we have to use the old session which triggered the deployment. - assertThat(shellRule.shell, - allOf( - hasSize(3), - new CommandLineMatcher() - .hasProlog('#!/bin/bash') - .hasSnippet('xs bg-deploy') - .hasOption('i', '1234') - .hasOption('a', 'resume'), - new CommandLineMatcher() - .hasProlog("#!/bin/bash") - .hasSnippet('xs logout'), - new CommandLineMatcher() - .hasProlog('') - .hasSnippet('rm \\$\\{XSCONFIG\\}') // delete the session file after logout - ) - ) - - assertThat(lockRule.getLockResources(), contains('xsDeploy:https://example.org/xs:myOrg:mySpace')) - - } - } \ No newline at end of file diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index d98033c1b..6cf3db0cd 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -1,38 +1,17 @@ -import com.sap.piper.JenkinsUtils - import static com.sap.piper.Prerequisites.checkScript -import com.sap.piper.BashUtils -import com.sap.piper.ConfigurationHelper +import com.sap.piper.JenkinsUtils +import com.sap.piper.PiperGoUtils + + import com.sap.piper.GenerateDocumentation import com.sap.piper.Utils import groovy.transform.Field -import hudson.AbortException - +@Field String METADATA_FILE = 'metadata/xsDeploy.yaml' @Field String STEP_NAME = getClass().getName() -@Field Set GENERAL_CONFIG_KEYS = STEP_CONFIG_KEYS - -@Field Set STEP_CONFIG_KEYS = [ - 'action', - 'apiUrl', - 'credentialsId', - 'deploymentId', - 'deployIdLogPattern', - 'deployOpts', - /** A map containing properties forwarded to dockerExecute. For more details see [here][dockerExecute] */ - 'docker', - 'loginOpts', - 'mode', - 'mtaPath', - 'org', - 'space', - 'xsSessionFile', -] - -@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS enum DeployMode { DEPLOY, @@ -67,272 +46,120 @@ void call(Map parameters = [:]) { handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: parameters) { + final script = checkScript(this, parameters) ?: null + + if(! script) { + error "Reference to surrounding pipeline script not provided (script: this)." + } + def utils = parameters.juStabUtils ?: new Utils() + def piperGoUtils = parameters.piperGoUtils ?: new PiperGoUtils(utils) - final script = checkScript(this, parameters) ?: this + // + // The parameters map in provided from outside. That map might be used elsewhere in the pipeline + // hence we should not modify it here. So we create a new map based on the parameters map. + parameters = [:] << parameters - ConfigurationHelper configHelper = ConfigurationHelper.newInstance(this) - .loadStepDefaults() - .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS) - .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS) - .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName?:env.STAGE_NAME, STEP_CONFIG_KEYS) - .addIfEmpty('mtaPath', script.commonPipelineEnvironment.getMtarFilePath()) - .addIfEmpty('deploymentId', script.commonPipelineEnvironment.xsDeploymentId) - .mixin(parameters, PARAMETER_KEYS) + // hard to predict how these two parameters looks like in its serialized form. Anyhow it is better + // not to have these parameters forwarded somehow to the go layer. + parameters.remove('juStabUtils') + parameters.remove('piperGoUtils') + parameters.remove('script') - Map config = configHelper.use() + // + // For now - since the xsDeploy step is not merged and covered by a release - we stash + // a locally built version of the piper-go binary in the pipeline script (Jenkinsfile) with + // stash name "piper-bin". That stash is used inside method "unstashPiperBin". + piperGoUtils.unstashPiperBin() - DeployMode mode = config.mode - - if(mode == DeployMode.NONE) { - echo "Deployment skipped intentionally. Deploy mode '${mode.toString()}'." - return - } - - Action action = config.action - - if(mode == DeployMode.DEPLOY && action != Action.NONE) { - error "Cannot perform action '${action.toString()}' in mode '${mode.toString()}'. Only action '${Action.NONE.toString()}' is allowed." - } - - boolean performLogin = ((mode == DeployMode.DEPLOY) || (mode == DeployMode.BG_DEPLOY && !(action in [Action.RESUME, Action.ABORT]))) - boolean performLogout = ((mode == DeployMode.DEPLOY) || (mode == DeployMode.BG_DEPLOY && action != Action.NONE)) - - boolean sessionExists = fileExists file: config.xsSessionFile - - if( (! performLogin) && (! sessionExists) ) { - error 'For the current configuration an already existing session is required. But there is no already existing session.' - } - - configHelper - .collectValidationFailures() - /** - * Used for finalizing the blue-green deployment. - * @possibleValues RESUME, ABORT, RETRY - */ - .withMandatoryProperty('action') - /** The file name of the file representing the sesssion after `xs login`. Should not be changed normally. */ - .withMandatoryProperty('xsSessionFile') - /** Regex pattern for retrieving the ID of the deployment. */ - .withMandatoryProperty('deployIdLogPattern') - /** - * Controls if there is a standard deployment or a blue green deployment - * @possibleValues DEPLOY, BG_DEPLOY - */ - .withMandatoryProperty('mode') - /** The endpoint */ - .withMandatoryProperty('apiUrl') - /** The organization */ - .withMandatoryProperty('org') - /** The space */ - .withMandatoryProperty('space') - /** Additional options appended to the login command. Only needed for sophisticated cases. - * When provided it is the duty of the provider to ensure proper quoting / escaping. - */ - .withMandatoryProperty('loginOpts') - /** Additional options appended to the deploy command. Only needed for sophisticated cases. - * When provided it is the duty of the provider to ensure proper quoting / escaping. - */ - .withMandatoryProperty('deployOpts') - /** The credentialsId */ - .withMandatoryProperty('credentialsId') - /** The path to the deployable. If not provided explicitly it is retrieved from the common pipeline environment - * (Parameter `mtarFilePath`). - */ - .withMandatoryProperty('mtaPath', null, {action == Action.NONE}) - .withMandatoryProperty('deploymentId', - 'No deployment id provided, neither via parameters nor via common pipeline environment. Was there a deployment before?', - {action in [Action.RESUME, Action.ABORT, Action.RETRY]}) - .use() + // + // Printing the piper-go version. Should not be done here, but somewhere during materializing + // the piper binary. + def piperGoVersion = sh(returnStdout: true, script: "./piper version") + echo "PiperGoVersion: ${piperGoVersion}" + // + // since there is no valid config provided (... null) telemetry is disabled. utils.pushToSWA([ step: STEP_NAME, - ], config) + ], null) - if(action == Action.NONE) { - boolean deployableExists = fileExists file: config.mtaPath - if(! deployableExists) - error "Deployable '${config.mtaPath}' does not exist." - } + writeFile(file: METADATA_FILE, text: libraryResource(METADATA_FILE)) - if(performLogin) { - login(script, config) - } - def failures = [] + withEnv([ + "PIPER_parametersJSON=${groovy.json.JsonOutput.toJson(parameters)}", + ]) { - if(action in [Action.RESUME, Action.ABORT, Action.RETRY]) { + // + // context config gives us e.g. the docker image name. --> How does this work for customer maintained images? + // There is a name provided in the metadata file. But we do not provide a docker image for that. + // The user has to build that for her/his own. How do we expect to configure this? + Map contextConfig = readJSON (text: sh(returnStdout: true, script: "./piper getConfig --contextConfig --stepMetadata '${METADATA_FILE}'")) - complete(script, mode, action, config, failures) + Map projectConfig = readJSON (text: sh(returnStdout: true, script: "./piper ${parameters.verbose ? '--verbose' :''} getConfig --stepMetadata '${METADATA_FILE}'")) - } else { + if(parameters.verbose) { + echo "[INFO] Context-Config: ${contextConfig}" + echo "[INFO] Project-Config: ${projectConfig}" + } - deploy(script, mode, config, failures) - } + Action action = projectConfig.action + DeployMode mode = projectConfig.mode - if (performLogout || failures) { - logout(script, config, failures) + // That config map here is only used in the groovy layer. Nothing is handed over to go. + Map config = contextConfig << + [ + apiUrl: projectConfig.apiUrl, // required on groovy level for acquire the lock + org: projectConfig.org, // required on groovy level for acquire the lock + space: projectConfig.space, // required on groovy level for acquire the lock + docker: [ + dockerImage: contextConfig.dockerImage, + dockerPullImage: false // dockerPullImage apparently not provided by context config. + ] + ] - } else { - echo "Skipping logout in order to be able to resume or abort later." - } + if(parameters.verbose) { + echo "[INFO] Merged-Config: ${config}" + } - if(failures) { - error "Failed command(s): ${failures}. Check earlier log for details." - } - } -} - -void login(Script script, Map config) { - - withCredentials([usernamePassword( - credentialsId: config.credentialsId, - passwordVariable: 'password', - usernameVariable: 'username' - )]) { - - def returnCode = executeXSCommand([script: script].plus(config.docker), - [ - "xs login -a ${config.apiUrl} -u ${username} -p ${BashUtils.quoteAndEscape(password)} -o ${config.org} -s ${config.space} ${config.loginOpts}", - 'RC=$?', - "[ \$RC == 0 ] && cp \"\${HOME}/${config.xsSessionFile}\" .", - 'exit $RC' - ]) - - if(returnCode != 0) - error "xs login failed." - } - - boolean existsXsSessionFileAfterLogin = fileExists file: config.xsSessionFile - if(! existsXsSessionFileAfterLogin) - error "Session file ${config.xsSessionFile} not found in current working directory after login." -} - -void deploy(Script script, DeployMode mode, Map config, def failures) { - - def deploymentLog - - try { - lock(getLockIdentifier(config)) { - deploymentLog = executeXSCommand([script: script].plus(config.docker), - [ - "cp ${config.xsSessionFile} \${HOME}", - "xs ${mode.toString()} '${config.mtaPath}' -f ${config.deployOpts}" - ], true) - } - - echo "Deploy log: ${deploymentLog}" - - } catch(AbortException e) { - echo "deployment failed. Message: ${e.getMessage()}, Log: ${deploymentLog}}" - failures << "xs ${mode.toString()}" - } - - if(mode == DeployMode.BG_DEPLOY) { - - if(! failures.isEmpty()) { - - echo "Retrieval of deploymentId skipped since prior deployment was not successfull." - - } else { - - for (def logLine : deploymentLog.readLines()) { - def matcher = logLine =~ config.deployIdLogPattern - if(matcher.find()) { - script.commonPipelineEnvironment.xsDeploymentId = matcher[0][1] - echo "DeploymentId: ${script.commonPipelineEnvironment.xsDeploymentId}." - break + def operationId + if(mode == DeployMode.BG_DEPLOY && action != Action.NONE) { + operationId = script.commonPipelineEnvironment.xsDeploymentId + if (! operationId) { + throw new IllegalArgumentException('No operationId provided. Was there a deployment before?') } } - if(script.commonPipelineEnvironment.xsDeploymentId == null) { - failures << "Cannot lookup deploymentId. Search pattern was: '${config.deployIdLogPattern}'." + + def xsDeployStdout + + lock(getLockIdentifier(config)) { + + withCredentials([usernamePassword( + credentialsId: config.credentialsId, + passwordVariable: 'PASSWORD', + usernameVariable: 'USERNAME')]) { + + dockerExecute([script: this].plus(config.docker)) { + xsDeployStdout = sh returnStdout: true, script: """#!/bin/bash + ./piper ${parameters.verbose ? '--verbose' : ''} xsDeploy --user \${USERNAME} --password \${PASSWORD} ${operationId ? "--operationId " + operationId : "" } + """ + } + + } + } + + if(mode == DeployMode.BG_DEPLOY && action == Action.NONE) { + script.commonPipelineEnvironment.xsDeploymentId = readJSON(text: xsDeployStdout).operationId + if (!script.commonPipelineEnvironment.xsDeploymentId) { + error "No Operation id returned from xs deploy step. This is required for mode '${mode}' and action '${action}'." + } + echo "[INFO] OperationId for subsequent resume or abort: '${script.commonPipelineEnvironment.xsDeploymentId}'." } } } } -void complete(Script script, DeployMode mode, Action action, Map config, def failures) { - - if(mode != DeployMode.BG_DEPLOY) - error "Action '${action.toString()}' can only be performed for mode '${DeployMode.BG_DEPLOY.toString()}'. Current mode is: '${mode.toString()}'." - - def returnCode = 1 - - lock(getLockIdentifier(config)) { - returnCode = executeXSCommand([script: script].plus(config.docker), - [ - "cp ${config.xsSessionFile} \${HOME}", - "xs ${mode.toString()} -i ${config.deploymentId} -a ${action.toString()}" - ]) - } - - if(returnCode != 0) { - echo "${mode.toString()} with action '${action.toString()}' failed with return code ${returnCode}." - failures << "xs ${mode.toString()} -a ${action.toString()}" - } -} - -void logout(Script script, Map config, def failures) { - - def returnCode = executeXSCommand([script: script].plus(config.docker), - [ - "cp ${config.xsSessionFile} \${HOME}", - 'xs logout' - ]) - - if(returnCode != 0) { - failures << 'xs logout' - } - - sh "XSCONFIG=${config.xsSessionFile}; [ -f \${XSCONFIG} ] && rm \${XSCONFIG}" -} - String getLockIdentifier(Map config) { "$STEP_NAME:${config.apiUrl}:${config.org}:${config.space}" } - -def executeXSCommand(Map dockerOptions, List commands, boolean returnStdout = false) { - - def r - - dockerExecute(dockerOptions) { - - // in case there are credentials contained in the commands we assume - // the call is properly wrapped by withCredentials(./.) - echo "Executing: '${commands}'." - - List prelude = [ - '#!/bin/bash' - ] - - List script = (prelude + commands) - - params = [ - script: script.join('\n') - ] - - if(returnStdout) { - params << [ returnStdout: true ] - } else { - params << [ returnStatus: true ] - } - - r = sh params - - if( (! returnStdout ) && r != 0) { - - try { - echo "xs logs:" - - sh 'LOG_FOLDER=${HOME}/.xs_logs; [ -d ${LOG_FOLDER} ] && cat ${LOG_FOLDER}/*' - - } catch(Exception e) { - - echo "Cannot provide xs logs: ${e.getMessage()}." - } - - echo "Executing of commands '${commands}' failed. Check earlier logs for details." - } - } - r -} From 264e7838338a5bbf916bba1ab9fd0c8c3d960724 Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Mon, 16 Dec 2019 09:20:15 +0100 Subject: [PATCH 02/10] Don't generate docu for xsDeploy from the source file expected to be done using the metadata file. --- vars/xsDeploy.groovy | 8 -------- 1 file changed, 8 deletions(-) diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index 6cf3db0cd..4548d8a22 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -34,14 +34,6 @@ enum Action { } } -/** - * Performs an XS deployment - * - * In case of blue-green deployments the step is called for the deployment in the narrower sense - * and later again for resuming or aborting. In this case both calls needs to be performed from the - * same directory. - */ -@GenerateDocumentation void call(Map parameters = [:]) { handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: parameters) { From e6b00fa601402039f0bd5628c5a676d90f6d3a6b Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Mon, 16 Dec 2019 10:40:44 +0100 Subject: [PATCH 03/10] Provide support for additional customer config layers. --- src/com/sap/piper/DefaultValueCache.groovy | 19 +++-- test/groovy/XsDeployTest.groovy | 83 +++++++++++++++++++--- vars/commonPipelineEnvironment.groovy | 5 ++ vars/xsDeploy.groovy | 24 ++++++- 4 files changed, 113 insertions(+), 18 deletions(-) diff --git a/src/com/sap/piper/DefaultValueCache.groovy b/src/com/sap/piper/DefaultValueCache.groovy index bab5ba47d..23c18d51f 100644 --- a/src/com/sap/piper/DefaultValueCache.groovy +++ b/src/com/sap/piper/DefaultValueCache.groovy @@ -8,16 +8,21 @@ class DefaultValueCache implements Serializable { private Map defaultValues - private DefaultValueCache(Map defaultValues){ + private List customDefaults = [] + + private DefaultValueCache(Map defaultValues, List customDefaults){ this.defaultValues = defaultValues + if(customDefaults) { + this.customDefaults.addAll(customDefaults) + } } static getInstance(){ return instance } - static createInstance(Map defaultValues){ - instance = new DefaultValueCache(defaultValues) + static createInstance(Map defaultValues, List customDefaults = []){ + instance = new DefaultValueCache(defaultValues, customDefaults) } Map getDefaultValues(){ @@ -28,6 +33,12 @@ class DefaultValueCache implements Serializable { instance = null } + List getCustomDefaults() { + def result = [] + result.addAll(customDefaults) + return result + } + static void prepare(Script steps, Map parameters = [:]) { if(parameters == null) parameters = [:] if(!DefaultValueCache.getInstance() || parameters.customDefaults) { @@ -46,7 +57,7 @@ class DefaultValueCache implements Serializable { MapUtils.pruneNulls(defaultValues), MapUtils.pruneNulls(configuration)) } - DefaultValueCache.createInstance(defaultValues) + DefaultValueCache.createInstance(defaultValues, customDefaults) } } } diff --git a/test/groovy/XsDeployTest.groovy b/test/groovy/XsDeployTest.groovy index a260bdc98..f06d3e1db 100644 --- a/test/groovy/XsDeployTest.groovy +++ b/test/groovy/XsDeployTest.groovy @@ -1,5 +1,6 @@ import static org.hamcrest.Matchers.allOf import static org.hamcrest.Matchers.contains +import static org.hamcrest.Matchers.containsInAnyOrder import static org.hamcrest.Matchers.containsString import static org.hamcrest.Matchers.equalTo import static org.hamcrest.Matchers.is @@ -63,8 +64,8 @@ class XsDeployTest extends BasePiperTest { @Before public void init() { helper.registerAllowedMethod('withEnv', [List, Closure], {l, c -> env = l; c()}) - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --contextConfig --stepMetadata.*', '{"dockerImage": "xs", "credentialsId":"myCreds"}') - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --stepMetadata.*', '{"mode": "BG_DEPLOY", "action": "NONE", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig.*--contextConfig.*', '{"dockerImage": "xs", "credentialsId":"myCreds"}') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'getConfig.* (?!--contextConfig)', '{"mode": "BG_DEPLOY", "action": "NONE", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') nullScript.commonPipelineEnvironment.xsDeploymentId = null } @@ -88,7 +89,7 @@ class XsDeployTest extends BasePiperTest { thrown.expect(IllegalArgumentException) thrown.expectMessage('No enum constant') - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --stepMetadata.*', '{"mode": "DOES_NOT_EXIST", "action": "NONE", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'getConfig.* (?!--contextConfig)', '{"mode": "DOES_NOT_EXIST", "action": "NONE", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') stepRule.step.xsDeploy( script: nullScript, @@ -114,7 +115,6 @@ class XsDeployTest extends BasePiperTest { piperGoUtils: goUtils ) - // nota bene: script and piperGoUtils are not contained in the json below. assertThat(env*.toString(), contains('PIPER_parametersJSON={"apiUrl":"https://example.org/xs","org":"myOrg","space":"mySpace","credentialsId":"myCreds","deployOpts":"-t 60","mtaPath":"myApp.mta","mode":"DEPLOY","action":"NONE"}')) } @@ -131,7 +131,7 @@ class XsDeployTest extends BasePiperTest { boolean unstashCalled assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, nullValue()) - + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*xsDeploy .*', '{"operationId": "1234"}') goUtils = new PiperGoUtils(null) { @@ -148,17 +148,21 @@ class XsDeployTest extends BasePiperTest { assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, is('1234')) - assertThat(writeFileRule.files.keySet(), contains('metadata/xsDeploy.yaml')) - + assertThat(writeFileRule.files.keySet(), containsInAnyOrder( + '.pipeline/additionalConfigs/default_pipeline_environment.yml', + 'metadata/xsDeploy.yaml', + )) + assertThat(dockerRule.dockerParams.dockerImage, equalTo('xs')) assertThat(dockerRule.dockerParams.dockerPullImage, equalTo(false)) - + assertThat(shellRule.shell, allOf( new CommandLineMatcher() .hasProlog('./piper version'), new CommandLineMatcher() - .hasProlog('./piper getConfig --contextConfig --stepMetadata \'metadata/xsDeploy.yaml\''), + .hasProlog('./piper getConfig') + .hasArgument('--contextConfig'), new CommandLineMatcher() .hasProlog('./piper getConfig --stepMetadata \'metadata/xsDeploy.yaml\''), new CommandLineMatcher() @@ -176,7 +180,7 @@ class XsDeployTest extends BasePiperTest { nullScript.commonPipelineEnvironment.xsDeploymentId = '1234' - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --stepMetadata.*', '{"mode": "BG_DEPLOY", "action": "RESUME", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'getConfig.* (?!--contextConfig)', '{"mode": "BG_DEPLOY", "action": "RESUME", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') stepRule.step.xsDeploy( script: nullScript, @@ -202,7 +206,7 @@ class XsDeployTest extends BasePiperTest { containsString('No operationId provided'), containsString('Was there a deployment before?'))) - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig --stepMetadata.*', '{"mode": "BG_DEPLOY", "action": "RESUME", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'getConfig.* (?!--contextConfig)', '{"mode": "BG_DEPLOY", "action": "RESUME", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, nullValue()) @@ -212,4 +216,61 @@ class XsDeployTest extends BasePiperTest { failOnError: true, ) } + + @Test + public void testAdditionalCustomConfigLayers() { + + def resources = ['a.yml': '- x: y}', 'b.yml' : '- a: b}'] + + helper.registerAllowedMethod('libraryResource', [String], { + + r -> + + def resource = resources[r] + if(resource) return resource + + File res = new File(new File('resources'), r) + if (res.exists()) { + return res.getText() + } + + throw new RuntimeException("Resource '${r}' not found.") + }) + + assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, nullValue()) + + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*xsDeploy .*', '{"operationId": "1234"}') + + nullScript.commonPipelineEnvironment = ['reset': {}, 'getCustomDefaults': {['a.yml', 'b.yml']}] + + goUtils = new PiperGoUtils(null) { + void unstashPiperBin() { + } + } + stepRule.step.xsDeploy( + script: nullScript, + piperGoUtils: goUtils + ) + + assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, is('1234')) + + assertThat(writeFileRule.files.keySet(), containsInAnyOrder( + '.pipeline/additionalConfigs/a.yml', + '.pipeline/additionalConfigs/b.yml', + '.pipeline/additionalConfigs/default_pipeline_environment.yml', + 'metadata/xsDeploy.yaml', + )) + + assertThat(shellRule.shell, + allOf( + new CommandLineMatcher() + .hasProlog('./piper getConfig') + .hasArgument('--contextConfig') + .hasArgument('--defaultConfig "b.yml" "a.yml" "default_pipeline_environment.yml"'), + new CommandLineMatcher() + .hasProlog('./piper getConfig --stepMetadata \'metadata/xsDeploy.yaml\''), + ) + ) + } + } \ No newline at end of file diff --git a/vars/commonPipelineEnvironment.groovy b/vars/commonPipelineEnvironment.groovy index a0ec24150..52b09ce94 100644 --- a/vars/commonPipelineEnvironment.groovy +++ b/vars/commonPipelineEnvironment.groovy @@ -1,5 +1,6 @@ import com.sap.piper.ConfigurationLoader import com.sap.piper.ConfigurationMerger +import com.sap.piper.DefaultValueCache import com.sap.piper.analytics.InfluxData class commonPipelineEnvironment implements Serializable { @@ -143,4 +144,8 @@ class commonPipelineEnvironment implements Serializable { config = ConfigurationMerger.merge(configuration.get('stages')?.get(stageName) ?: [:], null, config) return config } + List getCustomDefaults() { + DefaultValueCache.getInstance().getCustomDefaults() + } + } diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index 4548d8a22..6ba60252b 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -1,5 +1,6 @@ import static com.sap.piper.Prerequisites.checkScript +import com.sap.piper.DefaultValueCache import com.sap.piper.JenkinsUtils import com.sap.piper.PiperGoUtils @@ -10,6 +11,7 @@ import com.sap.piper.Utils import groovy.transform.Field @Field String METADATA_FILE = 'metadata/xsDeploy.yaml' +@Field String PIPER_DEFAULTS = 'default_pipeline_environment.yml' @Field String STEP_NAME = getClass().getName() @@ -76,8 +78,16 @@ void call(Map parameters = [:]) { step: STEP_NAME, ], null) - writeFile(file: METADATA_FILE, text: libraryResource(METADATA_FILE)) + List configs = [PIPER_DEFAULTS] + configs.addAll(script.commonPipelineEnvironment.getCustomDefaults()) + configs = configs.reverse() + + for(def customDefault : configs) { + writeFile(file: ".pipeline/additionalConfigs/${customDefault}", text: libraryResource(customDefault)) + } + + writeFile(file: METADATA_FILE, text: libraryResource(METADATA_FILE)) withEnv([ "PIPER_parametersJSON=${groovy.json.JsonOutput.toJson(parameters)}", @@ -87,9 +97,9 @@ void call(Map parameters = [:]) { // context config gives us e.g. the docker image name. --> How does this work for customer maintained images? // There is a name provided in the metadata file. But we do not provide a docker image for that. // The user has to build that for her/his own. How do we expect to configure this? - Map contextConfig = readJSON (text: sh(returnStdout: true, script: "./piper getConfig --contextConfig --stepMetadata '${METADATA_FILE}'")) + Map contextConfig = readJSON (text: sh(returnStdout: true, script: "./piper ${parameters.verbose ? '--verbose' :''} getConfig --stepMetadata '${METADATA_FILE}' --defaultConfig ${joinAndQuote(configs)} --contextConfig")) - Map projectConfig = readJSON (text: sh(returnStdout: true, script: "./piper ${parameters.verbose ? '--verbose' :''} getConfig --stepMetadata '${METADATA_FILE}'")) + Map projectConfig = readJSON (text: sh(returnStdout: true, script: "./piper ${parameters.verbose ? '--verbose' :''} getConfig --stepMetadata '${METADATA_FILE}' --defaultConfig ${joinAndQuote(configs)}")) if(parameters.verbose) { echo "[INFO] Context-Config: ${contextConfig}" @@ -155,3 +165,11 @@ void call(Map parameters = [:]) { String getLockIdentifier(Map config) { "$STEP_NAME:${config.apiUrl}:${config.org}:${config.space}" } + +String joinAndQuote(List l) { + _l = [] + for(def e : l) { + _l << '"' + e + '"' + } + _l.join(' ') +} From a1e093467e39d2e6fd677e73120077265e4873a3 Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Thu, 19 Dec 2019 08:56:10 +0100 Subject: [PATCH 04/10] put additional configs into dedicates folder --- test/groovy/XsDeployTest.groovy | 19 +++++----- vars/xsDeploy.groovy | 64 +++++++++++++++++++++------------ 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/test/groovy/XsDeployTest.groovy b/test/groovy/XsDeployTest.groovy index f06d3e1db..51e2d2ecc 100644 --- a/test/groovy/XsDeployTest.groovy +++ b/test/groovy/XsDeployTest.groovy @@ -150,7 +150,7 @@ class XsDeployTest extends BasePiperTest { assertThat(writeFileRule.files.keySet(), containsInAnyOrder( '.pipeline/additionalConfigs/default_pipeline_environment.yml', - 'metadata/xsDeploy.yaml', + '.pipeline/metadata/xsDeploy.yaml', )) assertThat(dockerRule.dockerParams.dockerImage, equalTo('xs')) @@ -164,11 +164,12 @@ class XsDeployTest extends BasePiperTest { .hasProlog('./piper getConfig') .hasArgument('--contextConfig'), new CommandLineMatcher() - .hasProlog('./piper getConfig --stepMetadata \'metadata/xsDeploy.yaml\''), + .hasProlog('./piper getConfig --stepMetadata \'.pipeline/metadata/xsDeploy.yaml\''), new CommandLineMatcher() - .hasProlog('#!/bin/bash ./piper xsDeploy --user \\$\\{USERNAME\\} --password \\$\\{PASSWORD\\}'), + .hasProlog('#!/bin/bash ./piper xsDeploy --defaultConfig ".pipeline/additionalConfigs/default_pipeline_environment.yml" --user \\$\\{USERNAME\\} --password \\$\\{PASSWORD\\}'), not(new CommandLineMatcher() - .hasProlog('#!/bin/bash ./piper xsDeploy --user \\$\\{USERNAME\\} --password \\$\\{PASSWORD\\} --operationId')) + .hasProlog('#!/bin/bash ./piper xsDeploy') + .hasOption('operationId', '1234')) ) ) @@ -189,7 +190,8 @@ class XsDeployTest extends BasePiperTest { assertThat(shellRule.shell, new CommandLineMatcher() - .hasProlog('#!/bin/bash ./piper xsDeploy --user \\$\\{USERNAME\\} --password \\$\\{PASSWORD\\} --operationId 1234') + .hasProlog('#!/bin/bash ./piper xsDeploy') + .hasOption('operationId', '1234') ) assertThat(lockRule.getLockResources(), contains('xsDeploy:https://example.org/xs:myOrg:mySpace')) @@ -258,7 +260,7 @@ class XsDeployTest extends BasePiperTest { '.pipeline/additionalConfigs/a.yml', '.pipeline/additionalConfigs/b.yml', '.pipeline/additionalConfigs/default_pipeline_environment.yml', - 'metadata/xsDeploy.yaml', + '.pipeline/metadata/xsDeploy.yaml', )) assertThat(shellRule.shell, @@ -266,11 +268,10 @@ class XsDeployTest extends BasePiperTest { new CommandLineMatcher() .hasProlog('./piper getConfig') .hasArgument('--contextConfig') - .hasArgument('--defaultConfig "b.yml" "a.yml" "default_pipeline_environment.yml"'), + .hasArgument('--defaultConfig ".pipeline/additionalConfigs/b.yml" ".pipeline/additionalConfigs/a.yml" ".pipeline/additionalConfigs/default_pipeline_environment.yml"'), new CommandLineMatcher() - .hasProlog('./piper getConfig --stepMetadata \'metadata/xsDeploy.yaml\''), + .hasProlog('./piper getConfig --stepMetadata \'.pipeline/metadata/xsDeploy.yaml\''), ) ) } - } \ No newline at end of file diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index 6ba60252b..f337da0c6 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -13,6 +13,8 @@ import groovy.transform.Field @Field String METADATA_FILE = 'metadata/xsDeploy.yaml' @Field String PIPER_DEFAULTS = 'default_pipeline_environment.yml' @Field String STEP_NAME = getClass().getName() +@Field String METADATA_FOLDER = '.pipeline' // metadata file contains already the "metadata" folder level, hence we end up in a folder ".pipeline/metadata" +@Field String ADDITIONAL_CONFIGS_FOLDER='.pipeline/additionalConfigs' enum DeployMode { @@ -54,40 +56,29 @@ void call(Map parameters = [:]) { // hence we should not modify it here. So we create a new map based on the parameters map. parameters = [:] << parameters - // hard to predict how these two parameters looks like in its serialized form. Anyhow it is better + // hard to predict how these parameters looks like in its serialized form. Anyhow it is better // not to have these parameters forwarded somehow to the go layer. parameters.remove('juStabUtils') parameters.remove('piperGoUtils') parameters.remove('script') - // - // For now - since the xsDeploy step is not merged and covered by a release - we stash - // a locally built version of the piper-go binary in the pipeline script (Jenkinsfile) with - // stash name "piper-bin". That stash is used inside method "unstashPiperBin". piperGoUtils.unstashPiperBin() // // Printing the piper-go version. Should not be done here, but somewhere during materializing - // the piper binary. + // the piper binary. As long as we don't have it elsewhere we should keep it here. def piperGoVersion = sh(returnStdout: true, script: "./piper version") echo "PiperGoVersion: ${piperGoVersion}" // - // since there is no valid config provided (... null) telemetry is disabled. + // since there is no valid config provided (... null) telemetry is disabled (same for other go releated steps at the moment). utils.pushToSWA([ step: STEP_NAME, ], null) + String configFiles = prepareConfigurations([PIPER_DEFAULTS].plus(script.commonPipelineEnvironment.getCustomDefaults()), ADDITIONAL_CONFIGS_FOLDER) - List configs = [PIPER_DEFAULTS] - configs.addAll(script.commonPipelineEnvironment.getCustomDefaults()) - configs = configs.reverse() - - for(def customDefault : configs) { - writeFile(file: ".pipeline/additionalConfigs/${customDefault}", text: libraryResource(customDefault)) - } - - writeFile(file: METADATA_FILE, text: libraryResource(METADATA_FILE)) + writeFile(file: "${METADATA_FOLDER}/${METADATA_FILE}", text: libraryResource(METADATA_FILE)) withEnv([ "PIPER_parametersJSON=${groovy.json.JsonOutput.toJson(parameters)}", @@ -97,9 +88,11 @@ void call(Map parameters = [:]) { // context config gives us e.g. the docker image name. --> How does this work for customer maintained images? // There is a name provided in the metadata file. But we do not provide a docker image for that. // The user has to build that for her/his own. How do we expect to configure this? - Map contextConfig = readJSON (text: sh(returnStdout: true, script: "./piper ${parameters.verbose ? '--verbose' :''} getConfig --stepMetadata '${METADATA_FILE}' --defaultConfig ${joinAndQuote(configs)} --contextConfig")) - Map projectConfig = readJSON (text: sh(returnStdout: true, script: "./piper ${parameters.verbose ? '--verbose' :''} getConfig --stepMetadata '${METADATA_FILE}' --defaultConfig ${joinAndQuote(configs)}")) + String projectConfigScript = "./piper ${parameters.verbose ? '--verbose' :''} getConfig --stepMetadata '${METADATA_FOLDER}/${METADATA_FILE}' --defaultConfig ${configFiles}" + String contextConfigScript = projectConfigScript + " --contextConfig" + Map projectConfig = readJSON (text: sh(returnStdout: true, script: projectConfigScript)) + Map contextConfig = readJSON (text: sh(returnStdout: true, script: contextConfigScript)) if(parameters.verbose) { echo "[INFO] Context-Config: ${contextConfig}" @@ -117,7 +110,7 @@ void call(Map parameters = [:]) { space: projectConfig.space, // required on groovy level for acquire the lock docker: [ dockerImage: contextConfig.dockerImage, - dockerPullImage: false // dockerPullImage apparently not provided by context config. + dockerPullImage: false ] ] @@ -144,7 +137,7 @@ void call(Map parameters = [:]) { dockerExecute([script: this].plus(config.docker)) { xsDeployStdout = sh returnStdout: true, script: """#!/bin/bash - ./piper ${parameters.verbose ? '--verbose' : ''} xsDeploy --user \${USERNAME} --password \${PASSWORD} ${operationId ? "--operationId " + operationId : "" } + ./piper ${parameters.verbose ? '--verbose' : ''} xsDeploy --defaultConfig ${configFiles} --user \${USERNAME} --password \${PASSWORD} ${operationId ? "--operationId " + operationId : "" } """ } @@ -166,10 +159,37 @@ String getLockIdentifier(Map config) { "$STEP_NAME:${config.apiUrl}:${config.org}:${config.space}" } -String joinAndQuote(List l) { +/* + * The returned string can be used directly in the command line for retrieving the configuration via go + */ +String prepareConfigurations(List configs, String configCacheFolder) { + + for(def customDefault : configs) { + writeFile(file: "${ADDITIONAL_CONFIGS_FOLDER}/${customDefault}", text: libraryResource(customDefault)) + } + joinAndQuote(configs.reverse(), configCacheFolder) +} + +/* + * prefix is supposed to be provided without trailing slash + */ +String joinAndQuote(List l, String prefix = '') { _l = [] + + if(prefix == null) { + prefix = '' + } + if(prefix.endsWith('/') || prefix.endsWith('\\')) + throw new IllegalArgumentException("Provide prefix (${prefix}) without trailing slash") + for(def e : l) { - _l << '"' + e + '"' + def _e = '' + if(prefix.length() > 0) { + _e += prefix + _e += '/' + } + _e += e + _l << '"' + _e + '"' } _l.join(' ') } From aba476a22d168cfc1046bb1ff1fbc059ce2a9eb7 Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Fri, 20 Dec 2019 11:51:43 +0100 Subject: [PATCH 05/10] Do not handover verbose flag explicitly to go binary It is sufficient to have the verbose flag in the configuration. --- vars/xsDeploy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index f337da0c6..651afe192 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -89,7 +89,7 @@ void call(Map parameters = [:]) { // There is a name provided in the metadata file. But we do not provide a docker image for that. // The user has to build that for her/his own. How do we expect to configure this? - String projectConfigScript = "./piper ${parameters.verbose ? '--verbose' :''} getConfig --stepMetadata '${METADATA_FOLDER}/${METADATA_FILE}' --defaultConfig ${configFiles}" + String projectConfigScript = "./piper getConfig --stepMetadata '${METADATA_FOLDER}/${METADATA_FILE}' --defaultConfig ${configFiles}" String contextConfigScript = projectConfigScript + " --contextConfig" Map projectConfig = readJSON (text: sh(returnStdout: true, script: projectConfigScript)) Map contextConfig = readJSON (text: sh(returnStdout: true, script: contextConfigScript)) @@ -137,7 +137,7 @@ void call(Map parameters = [:]) { dockerExecute([script: this].plus(config.docker)) { xsDeployStdout = sh returnStdout: true, script: """#!/bin/bash - ./piper ${parameters.verbose ? '--verbose' : ''} xsDeploy --defaultConfig ${configFiles} --user \${USERNAME} --password \${PASSWORD} ${operationId ? "--operationId " + operationId : "" } + ./piper xsDeploy --defaultConfig ${configFiles} --user \${USERNAME} --password \${PASSWORD} ${operationId ? "--operationId " + operationId : "" } """ } From fddfe4aab6cbdd0cdd8445414ab24a5918c9fd38 Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Fri, 20 Dec 2019 11:53:44 +0100 Subject: [PATCH 06/10] Streamline verbose logging --- vars/xsDeploy.groovy | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index 651afe192..ec739c41a 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -94,11 +94,6 @@ void call(Map parameters = [:]) { Map projectConfig = readJSON (text: sh(returnStdout: true, script: projectConfigScript)) Map contextConfig = readJSON (text: sh(returnStdout: true, script: contextConfigScript)) - if(parameters.verbose) { - echo "[INFO] Context-Config: ${contextConfig}" - echo "[INFO] Project-Config: ${projectConfig}" - } - Action action = projectConfig.action DeployMode mode = projectConfig.mode @@ -115,7 +110,7 @@ void call(Map parameters = [:]) { ] if(parameters.verbose) { - echo "[INFO] Merged-Config: ${config}" + echo "[INFO] Config: ${config}" } def operationId From dc86bad480b67d83c7fdf886b1236d487c8e099c Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Fri, 20 Dec 2019 14:22:39 +0100 Subject: [PATCH 07/10] get ride of the merged config --- vars/xsDeploy.groovy | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index ec739c41a..f8cfdd812 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -97,20 +97,9 @@ void call(Map parameters = [:]) { Action action = projectConfig.action DeployMode mode = projectConfig.mode - // That config map here is only used in the groovy layer. Nothing is handed over to go. - Map config = contextConfig << - [ - apiUrl: projectConfig.apiUrl, // required on groovy level for acquire the lock - org: projectConfig.org, // required on groovy level for acquire the lock - space: projectConfig.space, // required on groovy level for acquire the lock - docker: [ - dockerImage: contextConfig.dockerImage, - dockerPullImage: false - ] - ] - if(parameters.verbose) { - echo "[INFO] Config: ${config}" + echo "[INFO] ContextConfig: ${contextConfig}" + echo "[INFO] ProjectConfig: ${projectConfig}" } def operationId @@ -123,14 +112,14 @@ void call(Map parameters = [:]) { def xsDeployStdout - lock(getLockIdentifier(config)) { + lock(getLockIdentifier(projectConfig)) { withCredentials([usernamePassword( - credentialsId: config.credentialsId, + credentialsId: contextConfig.credentialsId, passwordVariable: 'PASSWORD', usernameVariable: 'USERNAME')]) { - dockerExecute([script: this].plus(config.docker)) { + dockerExecute([script: this].plus([dockerImage: contextConfig.dockerImage, dockerPullImage: false])) { xsDeployStdout = sh returnStdout: true, script: """#!/bin/bash ./piper xsDeploy --defaultConfig ${configFiles} --user \${USERNAME} --password \${PASSWORD} ${operationId ? "--operationId " + operationId : "" } """ From 8184312262f5a405a2ca64ad8a6ea92b72f6b868 Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Fri, 20 Dec 2019 14:41:17 +0100 Subject: [PATCH 08/10] Docker pull not hard coded anymore --- test/groovy/XsDeployTest.groovy | 2 +- vars/xsDeploy.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/groovy/XsDeployTest.groovy b/test/groovy/XsDeployTest.groovy index 51e2d2ecc..1e1bec77e 100644 --- a/test/groovy/XsDeployTest.groovy +++ b/test/groovy/XsDeployTest.groovy @@ -64,7 +64,7 @@ class XsDeployTest extends BasePiperTest { @Before public void init() { helper.registerAllowedMethod('withEnv', [List, Closure], {l, c -> env = l; c()}) - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig.*--contextConfig.*', '{"dockerImage": "xs", "credentialsId":"myCreds"}') + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '.*getConfig.*--contextConfig.*', '{"dockerImage": "xs", "dockerPullImage": false, "credentialsId":"myCreds"}') shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'getConfig.* (?!--contextConfig)', '{"mode": "BG_DEPLOY", "action": "NONE", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') nullScript.commonPipelineEnvironment.xsDeploymentId = null } diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index f8cfdd812..64b11d4c9 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -119,7 +119,7 @@ void call(Map parameters = [:]) { passwordVariable: 'PASSWORD', usernameVariable: 'USERNAME')]) { - dockerExecute([script: this].plus([dockerImage: contextConfig.dockerImage, dockerPullImage: false])) { + dockerExecute([script: this].plus([dockerImage: contextConfig.dockerImage, dockerPullImage: contextConfig.dockerPullImage])) { xsDeployStdout = sh returnStdout: true, script: """#!/bin/bash ./piper xsDeploy --defaultConfig ${configFiles} --user \${USERNAME} --password \${PASSWORD} ${operationId ? "--operationId " + operationId : "" } """ From a2b971845460451e1e7159248cce38cf8c95d782 Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Fri, 20 Dec 2019 14:49:57 +0100 Subject: [PATCH 09/10] Provide operationId also from signature --- test/groovy/XsDeployTest.groovy | 23 +++++++++++++++++++++++ vars/xsDeploy.groovy | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/test/groovy/XsDeployTest.groovy b/test/groovy/XsDeployTest.groovy index 1e1bec77e..ccb673a0a 100644 --- a/test/groovy/XsDeployTest.groovy +++ b/test/groovy/XsDeployTest.groovy @@ -219,6 +219,29 @@ class XsDeployTest extends BasePiperTest { ) } + @Test + public void testBlueGreenDeployResumeOperationIdViaSignature() { + + // this happens in case we would like to complete a deployment without having a (successful) deployments before. + + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'getConfig.* (?!--contextConfig)', '{"mode": "BG_DEPLOY", "action": "RESUME", "apiUrl": "https://example.org/xs", "org": "myOrg", "space": "mySpace"}') + + assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, nullValue()) + + stepRule.step.xsDeploy( + script: nullScript, + piperGoUtils: goUtils, + failOnError: true, + operationId: '1357' + ) + + assertThat(shellRule.shell, + new CommandLineMatcher() + .hasProlog('#!/bin/bash ./piper xsDeploy') + .hasOption('operationId', '1357') + ) + } + @Test public void testAdditionalCustomConfigLayers() { diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index 64b11d4c9..73cfc55ea 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -102,8 +102,8 @@ void call(Map parameters = [:]) { echo "[INFO] ProjectConfig: ${projectConfig}" } - def operationId - if(mode == DeployMode.BG_DEPLOY && action != Action.NONE) { + def operationId = parameters.operationId + if(! operationId && mode == DeployMode.BG_DEPLOY && action != Action.NONE) { operationId = script.commonPipelineEnvironment.xsDeploymentId if (! operationId) { throw new IllegalArgumentException('No operationId provided. Was there a deployment before?') From 6a90f81732024893c3970ef40557b2d188bf1b60 Mon Sep 17 00:00:00 2001 From: Marcus Holl Date: Tue, 7 Jan 2020 12:40:58 +0100 Subject: [PATCH 10/10] Fix code climat issues --- vars/xsDeploy.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vars/xsDeploy.groovy b/vars/xsDeploy.groovy index 73cfc55ea..aa38125d4 100644 --- a/vars/xsDeploy.groovy +++ b/vars/xsDeploy.groovy @@ -167,12 +167,12 @@ String joinAndQuote(List l, String prefix = '') { throw new IllegalArgumentException("Provide prefix (${prefix}) without trailing slash") for(def e : l) { - def _e = '' - if(prefix.length() > 0) { - _e += prefix - _e += '/' - } - _e += e + def _e = '' + if(prefix.length() > 0) { + _e += prefix + _e += '/' + } + _e += e _l << '"' + _e + '"' } _l.join(' ')