diff --git a/documentation/docs/steps/piperLoadGlobalExtensions.md b/documentation/docs/steps/piperLoadGlobalExtensions.md new file mode 100644 index 000000000..3d8f3171a --- /dev/null +++ b/documentation/docs/steps/piperLoadGlobalExtensions.md @@ -0,0 +1,9 @@ +# ${docGenStepName} + +## ${docGenDescription} + +## ${docGenParameters} + +## ${docGenConfiguration} + +## ${docJenkinsPluginDependencies} diff --git a/documentation/mkdocs.yml b/documentation/mkdocs.yml index 640c356b4..663a4d32c 100644 --- a/documentation/mkdocs.yml +++ b/documentation/mkdocs.yml @@ -82,6 +82,7 @@ nav: - pipelineStashFiles: steps/pipelineStashFiles.md - pipelineStashFilesAfterBuild: steps/pipelineStashFilesAfterBuild.md - pipelineStashFilesBeforeBuild: steps/pipelineStashFilesBeforeBuild.md + - piperLoadGlobalExtensions: steps/piperLoadGlobalExtensions.md - piperPublishWarnings: steps/piperPublishWarnings.md - prepareDefaultValues: steps/prepareDefaultValues.md - protecodeExecuteScan: steps/protecodeExecuteScan.md diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 63edaf12f..37f1f87dc 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -38,6 +38,7 @@ general: githubApiUrl: 'https://api.github.com' githubServerUrl: 'https://github.com' gitSshKeyCredentialsId: '' #needed to allow sshagent to run with local ssh key + globalExtensionsDirectory: '.pipeline/tmp/global_extensions/' jenkinsKubernetes: jnlpAgent: 'ppiper/jenkins-agent-k8s:v8' securityContext: @@ -580,7 +581,6 @@ steps: #defaults for stage wrapper piperStageWrapper: projectExtensionsDirectory: '.pipeline/extensions/' - globalExtensionsDirectory: '' stageLocking: true nodeLabel: '' stashContent: diff --git a/test/groovy/PiperLoadGlobalExtensionsTest.groovy b/test/groovy/PiperLoadGlobalExtensionsTest.groovy new file mode 100644 index 000000000..bfd3629bc --- /dev/null +++ b/test/groovy/PiperLoadGlobalExtensionsTest.groovy @@ -0,0 +1,158 @@ +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.RuleChain +import util.BasePiperTest +import util.JenkinsFileExistsRule +import util.JenkinsReadFileRule +import util.JenkinsReadYamlRule +import util.JenkinsStepRule +import util.JenkinsWriteFileRule +import util.Rules + +import static org.junit.Assert.assertEquals +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertNull +import static org.junit.Assert.assertTrue + +class PiperLoadGlobalExtensionsTest extends BasePiperTest { + + private Map checkoutParameters + private boolean checkoutCalled = false + private List filesRead = [] + private List fileWritten = [] + + private JenkinsStepRule stepRule = new JenkinsStepRule(this) + private JenkinsReadYamlRule readYamlRule = new JenkinsReadYamlRule(this) + private JenkinsFileExistsRule fileExistsRule = new JenkinsFileExistsRule(this, []) + + @Rule + public RuleChain ruleChain = Rules + .getCommonRules(this) + .around(stepRule) + .around(readYamlRule) + .around(fileExistsRule) + + @Before + void init() { + helper.registerAllowedMethod("checkout", [Map.class], { map -> + checkoutParameters = map + checkoutCalled = true + }) + helper.registerAllowedMethod("readFile", [Map.class], { map -> + filesRead.add(map.file) + return "" + }) + helper.registerAllowedMethod("writeFile", [Map.class], { map -> + fileWritten.add(map.file) + }) + } + + @Test + void testNotConfigured() throws Exception { + stepRule.step.piperLoadGlobalExtensions(script: nullScript) + assertFalse(checkoutCalled) + } + + @Test + void testUrlConfigured() throws Exception { + + nullScript.commonPipelineEnvironment.configuration = [ + general: [ + globalExtensionsRepository: 'https://my.git.example/foo/bar.git' + ] + ] + + stepRule.step.piperLoadGlobalExtensions(script: nullScript) + assertTrue(checkoutCalled) + assertEquals('GitSCM', checkoutParameters.$class) + assertEquals(1, checkoutParameters.userRemoteConfigs.size()) + assertEquals([url: 'https://my.git.example/foo/bar.git'], checkoutParameters.userRemoteConfigs[0]) + } + + @Test + void testVersionConfigured() throws Exception { + + nullScript.commonPipelineEnvironment.configuration = [ + general: [ + globalExtensionsRepository: 'https://my.git.example/foo/bar.git', + globalExtensionsVersion: 'v35' + ] + ] + + stepRule.step.piperLoadGlobalExtensions(script: nullScript) + assertTrue(checkoutCalled) + assertEquals(1, checkoutParameters.branches.size()) + assertEquals([name: 'v35'], checkoutParameters.branches[0]) + } + + @Test + void testCredentialsConfigured() throws Exception { + + nullScript.commonPipelineEnvironment.configuration = [ + general: [ + globalExtensionsRepository: 'https://my.git.example/foo/bar.git', + globalExtensionsRepositoryCredentialsId: 'my-credentials' + ] + ] + + stepRule.step.piperLoadGlobalExtensions(script: nullScript) + assertTrue(checkoutCalled) + assertEquals(1, checkoutParameters.userRemoteConfigs.size()) + assertEquals([url: 'https://my.git.example/foo/bar.git', credentialsId: 'my-credentials'], checkoutParameters.userRemoteConfigs[0]) + } + + @Test + void testExtensionConfigurationExists() throws Exception { + fileExistsRule.registerExistingFile('test/extension_configuration.yml') + + nullScript.commonPipelineEnvironment.configuration = [ + general: [ + globalExtensionsDirectory: 'test', + globalExtensionsRepository: 'https://my.git.example/foo/bar.git' + ] + ] + + Map prepareParameter = [:] + helper.registerAllowedMethod("prepareDefaultValues", [Map.class], { map -> + prepareParameter = map + }) + + stepRule.step.piperLoadGlobalExtensions(script: nullScript, customDefaults: ['default.yml'], customDefaultsFromFiles: ['file1.yml']) + assertTrue(checkoutCalled) + + //File copied + assertTrue(filesRead.contains('test/extension_configuration.yml')) + assertTrue(fileWritten.contains('.pipeline/extension_configuration.yml')) + + assertEquals(2, prepareParameter.customDefaultsFromFiles.size()) + assertEquals('extension_configuration.yml', prepareParameter.customDefaultsFromFiles[0]) + assertEquals('file1.yml', prepareParameter.customDefaultsFromFiles[1]) + assertEquals(1, prepareParameter.customDefaults.size()) + assertEquals('default.yml', prepareParameter.customDefaults[0]) + } + + @Test + void testLoadLibraries() throws Exception { + fileExistsRule.registerExistingFile('test/sharedLibraries.yml') + + nullScript.commonPipelineEnvironment.configuration = [ + general: [ + globalExtensionsDirectory: 'test', + globalExtensionsRepository: 'https://my.git.example/foo/bar.git' + ] + ] + + readYamlRule.registerYaml("test/sharedLibraries.yml", "[{name: my-extension-dependency, version: my-git-tag}]") + + List libsLoaded = [] + helper.registerAllowedMethod("library", [String.class], { lib -> + libsLoaded.add(lib) + }) + + stepRule.step.piperLoadGlobalExtensions(script: nullScript) + assertTrue(checkoutCalled) + assertEquals(1, libsLoaded.size()) + assertEquals("my-extension-dependency@my-git-tag", libsLoaded[0].toString()) + } +} diff --git a/test/groovy/PiperStageWrapperTest.groovy b/test/groovy/PiperStageWrapperTest.groovy index 9755cf84a..52fb3e77b 100644 --- a/test/groovy/PiperStageWrapperTest.groovy +++ b/test/groovy/PiperStageWrapperTest.groovy @@ -163,7 +163,7 @@ class PiperStageWrapperTest extends BasePiperTest { @Test void testGlobalOverwritingExtension() { helper.registerAllowedMethod('fileExists', [String.class], {s -> - return (s == 'test_global_overwriting.groovy') + return (s == '.pipeline/tmp/global_extensions/test_global_overwriting.groovy') }) helper.registerAllowedMethod('load', [String.class], { @@ -254,7 +254,7 @@ class PiperStageWrapperTest extends BasePiperTest { @Test void testStageCrashesInExtension() { helper.registerAllowedMethod('fileExists', [String.class], { path -> - return (path == 'test_crashing_extension.groovy') + return (path == '.pipeline/tmp/global_extensions/test_crashing_extension.groovy') }) helper.registerAllowedMethod('load', [String.class], { @@ -280,7 +280,7 @@ class PiperStageWrapperTest extends BasePiperTest { } assertThat(executed, is(true)) - assertThat(loggingRule.log, containsString('[piperStageWrapper] Found global interceptor \'test_crashing_extension.groovy\' for test_crashing_extension.')) + assertThat(loggingRule.log, containsString('[piperStageWrapper] Found global interceptor \'.pipeline/tmp/global_extensions/test_crashing_extension.groovy\' for test_crashing_extension.')) assertThat(DebugReport.instance.failedBuild.step, is('test_crashing_extension(extended)')) assertThat(DebugReport.instance.failedBuild.fatal, is('true')) assertThat(DebugReport.instance.failedBuild.reason, is(caught)) diff --git a/vars/piperLoadGlobalExtensions.groovy b/vars/piperLoadGlobalExtensions.groovy new file mode 100644 index 000000000..0e7a6877d --- /dev/null +++ b/vars/piperLoadGlobalExtensions.groovy @@ -0,0 +1,115 @@ +import com.sap.piper.ConfigurationHelper +import com.sap.piper.DebugReport +import com.sap.piper.GenerateDocumentation +import groovy.transform.Field + +import static com.sap.piper.Prerequisites.checkScript + +@Field String STEP_NAME = getClass().getName() + +@Field Set GENERAL_CONFIG_KEYS = [ + /** Directory where the extensions are cloned to*/ + 'globalExtensionsDirectory', + /** Git url of the repository containing the extensions*/ + 'globalExtensionsRepository', + /** Credentials required to clone the repository*/ + 'globalExtensionsRepositoryCredentialsId', + /** Version of the extensions which should be used, e.g. the tag name*/ + 'globalExtensionsVersion' +] + +@Field Set STEP_CONFIG_KEYS = [] + +@Field Set PARAMETER_KEYS = [ + /** This step will reinitialize the defaults. Make sure to pass the same customDefaults as to the step setupCommonPipelineEnvironment*/ + 'customDefaults', + /** This step will reinitialize the defaults. Make sure to pass the same customDefaultsFromFiles as to the step setupCommonPipelineEnvironment*/ + 'customDefaultsFromFiles' +] + +/** + * This step is part of the step setupCommonPipelineEnvironment and should not be used outside independently in a custom pipeline. + * This step allows users to define extensions (https://sap.github.io/jenkins-library/extensibility/#1-extend-individual-stages) globally instead of in each repository. + * Instead of defining the extensions in the .pipeline folder the extensions are defined in another repository. + * You can also place a file called extension_configuration.yml in this repository. + * Configuration defined in this file will be treated as default values with a lower precedence then custom defaults defined in the project configuration. + * You can also define additional Jenkins libraries these extensions depend on using a yaml file called sharedLibraries.yml: + * Example: + * - name: my-extension-dependency + * version: git-tag + */ +@GenerateDocumentation +void call(Map parameters = [:]) { + + handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: parameters) { + def script = checkScript(this, parameters) + // load default & individual configuration + Map configuration = ConfigurationHelper.newInstance(this) + .loadStepDefaults() + .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS) + .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS) + .mixin(parameters, PARAMETER_KEYS) + .use() + + if(!configuration.globalExtensionsRepository){ + return + } + + dir(configuration.globalExtensionsDirectory){ + Map gitParameters = [ + $class: 'GitSCM', + userRemoteConfigs: [[url: configuration.globalExtensionsRepository]] + ] + + if(configuration.globalExtensionsRepositoryCredentialsId){ + gitParameters.userRemoteConfigs[0].credentialsId = configuration.globalExtensionsRepositoryCredentialsId + } + + if(configuration.globalExtensionsVersion){ + gitParameters.branches = [[name: configuration.globalExtensionsVersion]] + } + + checkout(gitParameters) + } + + String extensionConfigurationFilePath = "${configuration.globalExtensionsDirectory}/extension_configuration.yml" + if (fileExists(extensionConfigurationFilePath)) { + writeFile file: ".pipeline/extension_configuration.yml", text: readFile(file: extensionConfigurationFilePath) + DebugReport.instance.globalExtensionConfigurationFilePath = extensionConfigurationFilePath + + prepareDefaultValues([ + script: script, + customDefaults: parameters.customDefaults, + customDefaultsFromFiles: ['extension_configuration.yml'] + parameters.customDefaultsFromFiles + ]) + } + + def globalExtensionsLibraryConfig = "${configuration.globalExtensionsDirectory}/sharedLibraries.yml" + + if(fileExists(globalExtensionsLibraryConfig)){ + loadLibrariesFromFile(globalExtensionsLibraryConfig) + } + } +} + +private loadLibrariesFromFile(String filename) { + List libs + try { + libs = readYaml file: filename + } + catch (Exception ex){ + error("Could not read extension libraries from ${filename}. The file has to contain a list of libraries where each entry should contain the name and the version of the library. (${ex.getMessage()})") + } + Set additionalLibraries = [] + for (int i = 0; i < libs.size(); i++) { + Map lib = libs[i] + String libName = lib.name + if(!libName){ + error("Could not read extension libraries from ${filename}. Each library definition has to have the field name defined.") + } + String branch = lib.version ?: 'master' + additionalLibraries.add("${libName} | ${branch}") + library "${libName}@${branch}" + } + DebugReport.instance.additionalSharedLibraries.addAll(additionalLibraries) +} diff --git a/vars/setupCommonPipelineEnvironment.groovy b/vars/setupCommonPipelineEnvironment.groovy index 4c70defe8..916083d75 100644 --- a/vars/setupCommonPipelineEnvironment.groovy +++ b/vars/setupCommonPipelineEnvironment.groovy @@ -80,6 +80,8 @@ void call(Map parameters = [:]) { customDefaults: parameters.customDefaults, customDefaultsFromFiles: customDefaultsFiles ]) + piperLoadGlobalExtensions script: script, customDefaults: parameters.customDefaults, customDefaultsFromFiles: customDefaultsFiles + stash name: 'pipelineConfigAndTests', includes: '.pipeline/**', allowEmpty: true Map config = ConfigurationHelper.newInstance(this)