Load global extensions in setupCommonPipelineEnvironment (#1688)

Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com>
Co-authored-by: Florian Wilhelm <florian.wilhelm02@sap.com>

documentation/docs/steps/piperLoadGlobalExtensions.md

@@ -0,0 +1,9 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}
+
+## ${docJenkinsPluginDependencies}

documentation/mkdocs.yml

@@ -82,6 +82,7 @@ nav:
         - pipelineStashFiles: steps/pipelineStashFiles.md
         - pipelineStashFilesAfterBuild: steps/pipelineStashFilesAfterBuild.md
         - pipelineStashFilesBeforeBuild: steps/pipelineStashFilesBeforeBuild.md
+        - piperLoadGlobalExtensions: steps/piperLoadGlobalExtensions.md
         - piperPublishWarnings: steps/piperPublishWarnings.md
         - prepareDefaultValues: steps/prepareDefaultValues.md
         - protecodeExecuteScan: steps/protecodeExecuteScan.md

resources/default_pipeline_environment.yml

@@ -38,6 +38,7 @@ general:
   githubApiUrl: 'https://api.github.com'
   githubServerUrl: 'https://github.com'
   gitSshKeyCredentialsId: '' #needed to allow sshagent to run with local ssh key
+  globalExtensionsDirectory: '.pipeline/tmp/global_extensions/'
   jenkinsKubernetes:
     jnlpAgent: 'ppiper/jenkins-agent-k8s:v8'
     securityContext:
@@ -580,7 +581,6 @@ steps:
   #defaults for stage wrapper
   piperStageWrapper:
     projectExtensionsDirectory: '.pipeline/extensions/'
-    globalExtensionsDirectory: ''
     stageLocking: true
     nodeLabel: ''
     stashContent:

test/groovy/PiperLoadGlobalExtensionsTest.groovy

@@ -0,0 +1,158 @@
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.RuleChain
+import util.BasePiperTest
+import util.JenkinsFileExistsRule
+import util.JenkinsReadFileRule
+import util.JenkinsReadYamlRule
+import util.JenkinsStepRule
+import util.JenkinsWriteFileRule
+import util.Rules
+
+import static org.junit.Assert.assertEquals
+import static org.junit.Assert.assertFalse
+import static org.junit.Assert.assertNull
+import static org.junit.Assert.assertTrue
+
+class PiperLoadGlobalExtensionsTest extends BasePiperTest {
+
+    private Map checkoutParameters
+    private boolean checkoutCalled = false
+    private List filesRead = []
+    private List fileWritten = []
+
+    private JenkinsStepRule stepRule = new JenkinsStepRule(this)
+    private JenkinsReadYamlRule readYamlRule = new JenkinsReadYamlRule(this)
+    private JenkinsFileExistsRule fileExistsRule = new JenkinsFileExistsRule(this, [])
+
+    @Rule
+    public RuleChain ruleChain = Rules
+        .getCommonRules(this)
+        .around(stepRule)
+        .around(readYamlRule)
+        .around(fileExistsRule)
+
+    @Before
+    void init() {
+        helper.registerAllowedMethod("checkout", [Map.class], { map ->
+            checkoutParameters = map
+            checkoutCalled = true
+        })
+        helper.registerAllowedMethod("readFile", [Map.class], { map ->
+            filesRead.add(map.file)
+            return ""
+        })
+        helper.registerAllowedMethod("writeFile", [Map.class], { map ->
+            fileWritten.add(map.file)
+        })
+    }
+
+    @Test
+    void testNotConfigured() throws Exception {
+        stepRule.step.piperLoadGlobalExtensions(script: nullScript)
+        assertFalse(checkoutCalled)
+    }
+
+    @Test
+    void testUrlConfigured() throws Exception {
+
+        nullScript.commonPipelineEnvironment.configuration = [
+            general: [
+                globalExtensionsRepository: 'https://my.git.example/foo/bar.git'
+            ]
+        ]
+
+        stepRule.step.piperLoadGlobalExtensions(script: nullScript)
+        assertTrue(checkoutCalled)
+        assertEquals('GitSCM', checkoutParameters.$class)
+        assertEquals(1, checkoutParameters.userRemoteConfigs.size())
+        assertEquals([url: 'https://my.git.example/foo/bar.git'], checkoutParameters.userRemoteConfigs[0])
+    }
+
+    @Test
+    void testVersionConfigured() throws Exception {
+
+        nullScript.commonPipelineEnvironment.configuration = [
+            general: [
+                globalExtensionsRepository: 'https://my.git.example/foo/bar.git',
+                globalExtensionsVersion: 'v35'
+            ]
+        ]
+
+        stepRule.step.piperLoadGlobalExtensions(script: nullScript)
+        assertTrue(checkoutCalled)
+        assertEquals(1, checkoutParameters.branches.size())
+        assertEquals([name: 'v35'], checkoutParameters.branches[0])
+    }
+
+    @Test
+    void testCredentialsConfigured() throws Exception {
+
+        nullScript.commonPipelineEnvironment.configuration = [
+            general: [
+                globalExtensionsRepository: 'https://my.git.example/foo/bar.git',
+                globalExtensionsRepositoryCredentialsId: 'my-credentials'
+            ]
+        ]
+
+        stepRule.step.piperLoadGlobalExtensions(script: nullScript)
+        assertTrue(checkoutCalled)
+        assertEquals(1, checkoutParameters.userRemoteConfigs.size())
+        assertEquals([url: 'https://my.git.example/foo/bar.git', credentialsId: 'my-credentials'], checkoutParameters.userRemoteConfigs[0])
+    }
+
+    @Test
+    void testExtensionConfigurationExists() throws Exception {
+        fileExistsRule.registerExistingFile('test/extension_configuration.yml')
+
+        nullScript.commonPipelineEnvironment.configuration = [
+            general: [
+                globalExtensionsDirectory: 'test',
+                globalExtensionsRepository: 'https://my.git.example/foo/bar.git'
+            ]
+        ]
+
+        Map prepareParameter = [:]
+        helper.registerAllowedMethod("prepareDefaultValues", [Map.class], { map ->
+            prepareParameter = map
+        })
+
+        stepRule.step.piperLoadGlobalExtensions(script: nullScript, customDefaults: ['default.yml'], customDefaultsFromFiles: ['file1.yml'])
+        assertTrue(checkoutCalled)
+
+        //File copied
+        assertTrue(filesRead.contains('test/extension_configuration.yml'))
+        assertTrue(fileWritten.contains('.pipeline/extension_configuration.yml'))
+
+        assertEquals(2, prepareParameter.customDefaultsFromFiles.size())
+        assertEquals('extension_configuration.yml', prepareParameter.customDefaultsFromFiles[0])
+        assertEquals('file1.yml', prepareParameter.customDefaultsFromFiles[1])
+        assertEquals(1, prepareParameter.customDefaults.size())
+        assertEquals('default.yml', prepareParameter.customDefaults[0])
+    }
+
+    @Test
+    void testLoadLibraries() throws Exception {
+        fileExistsRule.registerExistingFile('test/sharedLibraries.yml')
+
+        nullScript.commonPipelineEnvironment.configuration = [
+            general: [
+                globalExtensionsDirectory: 'test',
+                globalExtensionsRepository: 'https://my.git.example/foo/bar.git'
+            ]
+        ]
+
+        readYamlRule.registerYaml("test/sharedLibraries.yml", "[{name: my-extension-dependency, version: my-git-tag}]")
+
+        List libsLoaded = []
+        helper.registerAllowedMethod("library", [String.class], { lib ->
+            libsLoaded.add(lib)
+        })
+
+        stepRule.step.piperLoadGlobalExtensions(script: nullScript)
+        assertTrue(checkoutCalled)
+        assertEquals(1, libsLoaded.size())
+        assertEquals("my-extension-dependency@my-git-tag", libsLoaded[0].toString())
+    }
+}

test/groovy/PiperStageWrapperTest.groovy

@@ -163,7 +163,7 @@ class PiperStageWrapperTest extends BasePiperTest {
     @Test
     void testGlobalOverwritingExtension() {
         helper.registerAllowedMethod('fileExists', [String.class], {s ->
-            return (s == 'test_global_overwriting.groovy')
+            return (s == '.pipeline/tmp/global_extensions/test_global_overwriting.groovy')
         })
 
         helper.registerAllowedMethod('load', [String.class], {
@@ -254,7 +254,7 @@ class PiperStageWrapperTest extends BasePiperTest {
     @Test
     void testStageCrashesInExtension() {
         helper.registerAllowedMethod('fileExists', [String.class], { path ->
-            return (path == 'test_crashing_extension.groovy')
+            return (path == '.pipeline/tmp/global_extensions/test_crashing_extension.groovy')
         })
 
         helper.registerAllowedMethod('load', [String.class], {
@@ -280,7 +280,7 @@ class PiperStageWrapperTest extends BasePiperTest {
         }
 
         assertThat(executed, is(true))
-        assertThat(loggingRule.log, containsString('[piperStageWrapper] Found global interceptor \'test_crashing_extension.groovy\' for test_crashing_extension.'))
+        assertThat(loggingRule.log, containsString('[piperStageWrapper] Found global interceptor \'.pipeline/tmp/global_extensions/test_crashing_extension.groovy\' for test_crashing_extension.'))
         assertThat(DebugReport.instance.failedBuild.step, is('test_crashing_extension(extended)'))
         assertThat(DebugReport.instance.failedBuild.fatal, is('true'))
         assertThat(DebugReport.instance.failedBuild.reason, is(caught))

vars/piperLoadGlobalExtensions.groovy

@@ -0,0 +1,115 @@
+import com.sap.piper.ConfigurationHelper
+import com.sap.piper.DebugReport
+import com.sap.piper.GenerateDocumentation
+import groovy.transform.Field
+
+import static com.sap.piper.Prerequisites.checkScript
+
+@Field String STEP_NAME = getClass().getName()
+
+@Field Set GENERAL_CONFIG_KEYS = [
+    /** Directory where the extensions are cloned to*/
+    'globalExtensionsDirectory',
+    /** Git url of the repository containing the extensions*/
+    'globalExtensionsRepository',
+    /** Credentials required to clone the repository*/
+    'globalExtensionsRepositoryCredentialsId',
+    /** Version of the extensions which should be used, e.g. the tag name*/
+    'globalExtensionsVersion'
+]
+
+@Field Set STEP_CONFIG_KEYS = []
+
+@Field Set PARAMETER_KEYS = [
+    /** This step will reinitialize the defaults. Make sure to pass the same customDefaults as to the step setupCommonPipelineEnvironment*/
+    'customDefaults',
+    /** This step will reinitialize the defaults. Make sure to pass the same customDefaultsFromFiles as to the step setupCommonPipelineEnvironment*/
+    'customDefaultsFromFiles'
+]
+
+/**
+ * This step is part of the step setupCommonPipelineEnvironment and should not be used outside independently in a custom pipeline.
+ * This step allows users to define extensions (https://sap.github.io/jenkins-library/extensibility/#1-extend-individual-stages) globally instead of in each repository.
+ * Instead of defining the extensions in the .pipeline folder the extensions are defined in another repository.
+ * You can also place a file called extension_configuration.yml in this repository.
+ * Configuration defined in this file will be treated as default values with a lower precedence then custom defaults defined in the project configuration.
+ * You can also define additional Jenkins libraries these extensions depend on using a yaml file called sharedLibraries.yml:
+ * Example:
+ * - name: my-extension-dependency
+ *   version: git-tag
+ */
+@GenerateDocumentation
+void call(Map parameters = [:]) {
+
+    handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: parameters) {
+        def script = checkScript(this, parameters)
+        // load default & individual configuration
+        Map configuration = ConfigurationHelper.newInstance(this)
+            .loadStepDefaults()
+            .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS)
+            .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS)
+            .mixin(parameters, PARAMETER_KEYS)
+            .use()
+
+        if(!configuration.globalExtensionsRepository){
+            return
+        }
+
+        dir(configuration.globalExtensionsDirectory){
+            Map gitParameters = [
+                $class: 'GitSCM',
+                userRemoteConfigs: [[url: configuration.globalExtensionsRepository]]
+            ]
+
+            if(configuration.globalExtensionsRepositoryCredentialsId){
+                gitParameters.userRemoteConfigs[0].credentialsId = configuration.globalExtensionsRepositoryCredentialsId
+            }
+
+            if(configuration.globalExtensionsVersion){
+                gitParameters.branches = [[name: configuration.globalExtensionsVersion]]
+            }
+
+            checkout(gitParameters)
+        }
+
+        String extensionConfigurationFilePath = "${configuration.globalExtensionsDirectory}/extension_configuration.yml"
+        if (fileExists(extensionConfigurationFilePath)) {
+            writeFile file: ".pipeline/extension_configuration.yml", text: readFile(file: extensionConfigurationFilePath)
+            DebugReport.instance.globalExtensionConfigurationFilePath = extensionConfigurationFilePath
+
+            prepareDefaultValues([
+                script: script,
+                customDefaults: parameters.customDefaults,
+                customDefaultsFromFiles: ['extension_configuration.yml'] + parameters.customDefaultsFromFiles
+            ])
+        }
+
+        def globalExtensionsLibraryConfig = "${configuration.globalExtensionsDirectory}/sharedLibraries.yml"
+
+        if(fileExists(globalExtensionsLibraryConfig)){
+            loadLibrariesFromFile(globalExtensionsLibraryConfig)
+        }
+    }
+}
+
+private loadLibrariesFromFile(String filename) {
+    List libs
+    try {
+        libs = readYaml file: filename
+    }
+    catch (Exception ex){
+        error("Could not read extension libraries from ${filename}. The file has to contain a list of libraries where each entry should contain the name and the version of the library. (${ex.getMessage()})")
+    }
+    Set additionalLibraries = []
+    for (int i = 0; i < libs.size(); i++) {
+        Map lib = libs[i]
+        String libName = lib.name
+        if(!libName){
+            error("Could not read extension libraries from ${filename}. Each library definition has to have the field name defined.")
+        }
+        String branch = lib.version ?: 'master'
+        additionalLibraries.add("${libName} | ${branch}")
+        library "${libName}@${branch}"
+    }
+    DebugReport.instance.additionalSharedLibraries.addAll(additionalLibraries)
+}

vars/setupCommonPipelineEnvironment.groovy

@@ -80,6 +80,8 @@ void call(Map parameters = [:]) {
             customDefaults: parameters.customDefaults,
             customDefaultsFromFiles: customDefaultsFiles ])
 
+        piperLoadGlobalExtensions script: script, customDefaults: parameters.customDefaults, customDefaultsFromFiles: customDefaultsFiles
+
         stash name: 'pipelineConfigAndTests', includes: '.pipeline/**', allowEmpty: true
 
         Map config = ConfigurationHelper.newInstance(this)