Add buildDescriptorExcludeList parameter to npmExecuteScripts step (#1800)

This change adds a buildDescriptorExcludeList parameter to
npmExecuteScripts, to enable the exclusion of certain directories when
executing npm scripts. Previously, npmExecuteScripts could only execute
scripts in all packages.

Now it is possible to provide paths or patterns as elements of the
buildDescriptorExcludeList to exclude packages when executing npm scripts.

cmd/npmExecuteLint.go

@@ -100,7 +100,7 @@ func runCiLint(npmExecutor npm.Executor, failOnError bool) error {
 	runScripts := []string{"ci-lint"}
 	runOptions := []string{"--silent"}
 
-	err := npmExecutor.RunScriptsInAllPackages(runScripts, runOptions, nil, false)
+	err := npmExecutor.RunScriptsInAllPackages(runScripts, runOptions, nil, false, nil)
 	if err != nil {
 		if failOnError {
 			return fmt.Errorf("ci-lint script execution failed with error: %w. This might be the result of severe linting findings, or some other issue while executing the script. Please examine the linting results in the UI, the cilint.xml file, if available, or the log above. ", err)

cmd/npmExecuteScripts.go

@@ -17,14 +17,17 @@ func npmExecuteScripts(config npmExecuteScriptsOptions, telemetryData *telemetry
 }
 
 func runNpmExecuteScripts(npmExecutor npm.Executor, config *npmExecuteScriptsOptions) error {
-	packageJSONFiles := npmExecutor.FindPackageJSONFiles()
-
 	if config.Install {
-		err := npmExecutor.InstallAllDependencies(packageJSONFiles)
+		packageJSONFiles, err := npmExecutor.FindPackageJSONFilesWithExcludes(config.BuildDescriptorExcludeList)
+		if err != nil {
+			return err
+		}
+
+		err = npmExecutor.InstallAllDependencies(packageJSONFiles)
 		if err != nil {
 			return err
 		}
 	}
 
-	return npmExecutor.RunScriptsInAllPackages(config.RunScripts, nil, config.ScriptOptions, config.VirtualFrameBuffer)
+	return npmExecutor.RunScriptsInAllPackages(config.RunScripts, nil, config.ScriptOptions, config.VirtualFrameBuffer, config.BuildDescriptorExcludeList)
 }

cmd/npmExecuteScripts_generated.go

@@ -20,6 +20,7 @@ type npmExecuteScriptsOptions struct {
 	SapNpmRegistry             string   `json:"sapNpmRegistry,omitempty"`
 	VirtualFrameBuffer         bool     `json:"virtualFrameBuffer,omitempty"`
 	ScriptOptions              []string `json:"scriptOptions,omitempty"`
+	BuildDescriptorExcludeList []string `json:"buildDescriptorExcludeList,omitempty"`
 }
 
 // NpmExecuteScriptsCommand Execute npm run scripts on all npm packages in a project
@@ -83,6 +84,7 @@ func addNpmExecuteScriptsFlags(cmd *cobra.Command, stepConfig *npmExecuteScripts
 	cmd.Flags().StringVar(&stepConfig.SapNpmRegistry, "sapNpmRegistry", `https://npm.sap.com`, "The default npm registry URL to be used as the remote mirror for the SAP npm packages.")
 	cmd.Flags().BoolVar(&stepConfig.VirtualFrameBuffer, "virtualFrameBuffer", false, "(Linux only) Start a virtual frame buffer in the background. This allows you to run a web browser without the need for an X server. Note that xvfb needs to be installed in the execution environment.")
 	cmd.Flags().StringSliceVar(&stepConfig.ScriptOptions, "scriptOptions", []string{}, "Options are passed to all runScripts calls separated by a '--'. './piper npmExecuteScripts --runScripts ci-e2e --scriptOptions '--tag1' will correspond to 'npm run ci-e2e -- --tag1'")
+	cmd.Flags().StringSliceVar(&stepConfig.BuildDescriptorExcludeList, "buildDescriptorExcludeList", []string{`deployment/**`}, "List of build descriptors and therefore modules to exclude from execution of the npm scripts. The elements can either be a path to the build descriptor or a pattern.")
 
 }
 
@@ -144,6 +146,14 @@ func npmExecuteScriptsMetadata() config.StepData {
 						Mandatory:   false,
 						Aliases:     []config.Alias{},
 					},
+					{
+						Name:        "buildDescriptorExcludeList",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "[]string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+					},
 				},
 			},
 		},

cmd/npmExecuteScripts_test.go

@@ -32,6 +32,7 @@ type npmConfig struct {
 	runOptions         []string
 	scriptOptions      []string
 	virtualFrameBuffer bool
+	excludeList        []string
 }
 
 // npmExecutorMock mocking struct
@@ -46,13 +47,19 @@ func (n *npmExecutorMock) FindPackageJSONFiles() []string {
 	return packages
 }
 
+// FindPackageJSONFiles mock implementation
+func (n *npmExecutorMock) FindPackageJSONFilesWithExcludes(excludeList []string) ([]string, error) {
+	packages, _ := n.utils.Glob("**/package.json")
+	return packages, nil
+}
+
 // FindPackageJSONFilesWithScript mock implementation
 func (n *npmExecutorMock) FindPackageJSONFilesWithScript(packageJSONFiles []string, script string) ([]string, error) {
 	return packageJSONFiles, nil
 }
 
 // RunScriptsInAllPackages mock implementation
-func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool) error {
+func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error {
 	if len(runScripts) != len(n.config.runScripts) {
 		return fmt.Errorf("RunScriptsInAllPackages was called with a different list of runScripts than config.runScripts")
 	}
@@ -63,7 +70,7 @@ func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOption
 	}
 
 	if len(scriptOptions) != len(n.config.scriptOptions) {
-		return fmt.Errorf("RunScriptsInAllPackages was called with a different list of runOptions than config.scriptOptions")
+		return fmt.Errorf("RunScriptsInAllPackages was called with a different list of scriptOptions than config.scriptOptions")
 	}
 
 	if len(runOptions) != len(n.config.runOptions) {
@@ -74,6 +81,10 @@ func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOption
 		return fmt.Errorf("RunScriptsInAllPackages was called with a different value of virtualFrameBuffer than config.virtualFrameBuffer")
 	}
 
+	if len(excludeList) != len(n.config.excludeList) {
+		return fmt.Errorf("RunScriptsInAllPackages was called with a different value of excludeList than config.excludeList")
+	}
+
 	return nil
 }
 
@@ -101,6 +112,18 @@ func (n *npmExecutorMock) SetNpmRegistries() error {
 }
 
 func TestNpmExecuteScripts(t *testing.T) {
+	t.Run("Call with excludeList", func(t *testing.T) {
+		config := npmExecuteScriptsOptions{Install: true, RunScripts: []string{"ci-build", "ci-test"}, BuildDescriptorExcludeList: []string{"**/path/**"}}
+		utils := newNpmMockUtilsBundle()
+		utils.AddFile("package.json", []byte("{\"name\": \"Test\" }"))
+		utils.AddFile("src/package.json", []byte("{\"name\": \"Test\" }"))
+
+		npmExecutor := npmExecutorMock{utils: utils, config: npmConfig{install: config.Install, runScripts: config.RunScripts, excludeList: config.BuildDescriptorExcludeList}}
+		err := runNpmExecuteScripts(&npmExecutor, &config)
+
+		assert.NoError(t, err)
+	})
+
 	t.Run("Call with scriptOptions", func(t *testing.T) {
 		config := npmExecuteScriptsOptions{Install: true, RunScripts: []string{"ci-build", "ci-test"}, ScriptOptions: []string{"--run"}}
 		utils := newNpmMockUtilsBundle()

pkg/npm/npm.go

@@ -7,8 +7,8 @@ import (
 	"github.com/SAP/jenkins-library/pkg/command"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
+	"github.com/bmatcuk/doublestar"
 	"io"
-	"os"
 	"path/filepath"
 	"strings"
 )
@@ -22,8 +22,9 @@ type Execute struct {
 // Executor interface to enable mocking for testing
 type Executor interface {
 	FindPackageJSONFiles() []string
+	FindPackageJSONFilesWithExcludes(excludeList []string) ([]string, error)
 	FindPackageJSONFilesWithScript(packageJSONFiles []string, script string) ([]string, error)
-	RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool) error
+	RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error
 	InstallAllDependencies(packageJSONFiles []string) error
 	SetNpmRegistries() error
 }
@@ -128,8 +129,12 @@ func registryRequiresConfiguration(preConfiguredRegistry, url string) bool {
 }
 
 // RunScriptsInAllPackages runs all scripts defined in ExecuteOptions.RunScripts
-func (exec *Execute) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool) error {
+func (exec *Execute) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error {
-	packageJSONFiles := exec.FindPackageJSONFiles()
+	packageJSONFiles, err := exec.FindPackageJSONFilesWithExcludes(excludeList)
+	if err != nil {
+		return err
+	}
+
 	execRunner := exec.Utils.GetExecRunner()
 
 	if virtualFrameBuffer {
@@ -205,25 +210,42 @@ func (exec *Execute) executeScript(packageJSON string, script string, runOptions
 	return nil
 }
 
-// FindPackageJSONFiles returns a list of all package.json fileUtils of the project excluding node_modules and gen/ directories
+// FindPackageJSONFiles returns a list of all package.json files of the project excluding node_modules and gen/ directories
 func (exec *Execute) FindPackageJSONFiles() []string {
+	packageJSONFiles, _ := exec.FindPackageJSONFilesWithExcludes([]string{})
+	return packageJSONFiles
+}
+
+// FindPackageJSONFilesWithExcludes returns a list of all package.json files of the project excluding node_modules, gen/ and directories/patterns defined by excludeList
+func (exec *Execute) FindPackageJSONFilesWithExcludes(excludeList []string) ([]string, error) {
 	unfilteredListOfPackageJSONFiles, _ := exec.Utils.Glob("**/package.json")
 
+	nodeModulesExclude := "**/node_modules/**"
+	genExclude := "**/gen/**"
+	excludeList = append(excludeList, nodeModulesExclude, genExclude)
+
 	var packageJSONFiles []string
 
 	for _, file := range unfilteredListOfPackageJSONFiles {
-		if strings.Contains(file, "node_modules") {
+		excludePackage := false
-			continue
+		for _, exclude := range excludeList {
+			matched, err := doublestar.PathMatch(exclude, file)
+			if err != nil {
+				return nil, fmt.Errorf("failed to match file %s to pattern %s: %w", file, exclude, err)
 			}
-
+			if matched {
-		if strings.HasPrefix(file, "gen"+string(os.PathSeparator)) || strings.Contains(file, string(os.PathSeparator)+"gen"+string(os.PathSeparator)) {
+				excludePackage = true
+				break
+			}
+		}
+		if excludePackage {
 			continue
 		}
 
 		packageJSONFiles = append(packageJSONFiles, file)
 		log.Entry().Info("Discovered package.json file " + file)
 	}
-	return packageJSONFiles
+	return packageJSONFiles, nil
 }
 
 // FindPackageJSONFilesWithScript returns a list of package.json fileUtils that contain the script

pkg/npm/npm_test.go

@@ -36,13 +36,16 @@ func TestNpm(t *testing.T) {
 		packageJSONFiles := exec.FindPackageJSONFiles()
 
 		assert.Equal(t, []string{"package.json"}, packageJSONFiles)
+
 	})
 
-	t.Run("find package.json files with two package.json and filtered package.json", func(t *testing.T) {
+	t.Run("find package.json files with two package.json and default filter", func(t *testing.T) {
 		utils := newNpmMockUtilsBundle()
 		utils.AddFile("package.json", []byte("{}"))
-		utils.AddFile(filepath.Join("src", "package.json"), []byte("{}"))
+		utils.AddFile(filepath.Join("src", "package.json"), []byte("{}"))          // should NOT be filtered out
 		utils.AddFile(filepath.Join("node_modules", "package.json"), []byte("{}")) // is filtered out
+		utils.AddFile(filepath.Join("gen", "package.json"), []byte("{}"))          // is filtered out
+
 		options := ExecutorOptions{}
 
 		exec := &Execute{
@@ -55,6 +58,32 @@ func TestNpm(t *testing.T) {
 		assert.Equal(t, []string{"package.json", filepath.Join("src", "package.json")}, packageJSONFiles)
 	})
 
+	t.Run("find package.json files with two package.json and excludes", func(t *testing.T) {
+		utils := newNpmMockUtilsBundle()
+		utils.AddFile("package.json", []byte("{}"))
+		utils.AddFile(filepath.Join("src", "package.json"), []byte("{}"))                  // should NOT be filtered out
+		utils.AddFile(filepath.Join("notfiltered", "package.json"), []byte("{}"))          // should NOT be filtered out
+		utils.AddFile(filepath.Join("Path", "To", "filter", "package.json"), []byte("{}")) // should NOT be filtered out
+		utils.AddFile(filepath.Join("node_modules", "package.json"), []byte("{}"))         // is filtered out
+		utils.AddFile(filepath.Join("gen", "package.json"), []byte("{}"))                  // is filtered out
+		utils.AddFile(filepath.Join("filter", "package.json"), []byte("{}"))               // is filtered out
+		utils.AddFile(filepath.Join("filterPath", "package.json"), []byte("{}"))           // is filtered out
+		utils.AddFile(filepath.Join("filter", "Path", "To", "package.json"), []byte("{}")) // is filtered out
+
+		options := ExecutorOptions{}
+
+		exec := &Execute{
+			Utils:   &utils,
+			Options: options,
+		}
+
+		packageJSONFiles, err := exec.FindPackageJSONFilesWithExcludes([]string{"filter/**", "filterPath/package.json"})
+
+		if assert.NoError(t, err) {
+			assert.Equal(t, []string{filepath.Join("Path", "To", "filter", "package.json"), filepath.Join("notfiltered", "package.json"), "package.json", filepath.Join("src", "package.json")}, packageJSONFiles)
+		}
+	})
+
 	t.Run("find package.json files with script", func(t *testing.T) {
 		utils := newNpmMockUtilsBundle()
 		utils.AddFile("package.json", []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }"))
@@ -231,7 +260,7 @@ func TestNpm(t *testing.T) {
 			Utils:   &utils,
 			Options: options,
 		}
-		err := exec.RunScriptsInAllPackages(runScripts, nil, nil, false)
+		err := exec.RunScriptsInAllPackages(runScripts, nil, nil, false, nil)
 
 		if assert.NoError(t, err) {
 			if assert.Equal(t, 6, len(utils.execRunner.Calls)) {
@@ -295,7 +324,7 @@ func TestNpm(t *testing.T) {
 			Utils:   &utils,
 			Options: options,
 		}
-		err := exec.RunScriptsInAllPackages([]string{"foo"}, nil, nil, true)
+		err := exec.RunScriptsInAllPackages([]string{"foo"}, nil, nil, true, nil)
 
 		assert.Contains(t, utils.execRunner.Env, "DISPLAY=:99")
 		assert.NoError(t, err)

resources/metadata/npmExecuteScripts.yaml

@@ -58,6 +58,14 @@ spec:
           - PARAMETERS
           - STAGES
           - STEPS
+      - name: buildDescriptorExcludeList
+        type: '[]string'
+        description: List of build descriptors and therefore modules to exclude from execution of the npm scripts. The elements can either be a path to the build descriptor or a pattern.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        default: ["deployment/**"]
   containers:
     - name: node
       image: node:12-buster-slim

test/groovy/NpmExecuteEndToEndTestsTest.groovy

@@ -223,6 +223,7 @@ class NpmExecuteEndToEndTestsTest extends BasePiperTest {
         stepRule.step.npmExecuteEndToEndTests(
             script: nullScript,
             stage: "myStage",
+            buildDescriptorExcludeList: ["path/to/package.json"],
             runScript: "ci-e2e"
         )
 
@@ -232,6 +233,7 @@ class NpmExecuteEndToEndTestsTest extends BasePiperTest {
         assert npmExecuteScriptsRule.hasParameter('virtualFrameBuffer', true)
         assert npmExecuteScriptsRule.hasParameter('runScripts', ["ci-e2e"])
         assert npmExecuteScriptsRule.hasParameter('scriptOptions', ["--launchUrl=${appUrl.url}"] + appUrl.parameters)
+        assert npmExecuteScriptsRule.hasParameter('buildDescriptorExcludeList', ["path/to/package.json"])
     }
 
     @Test

vars/npmExecuteEndToEndTests.groovy

@@ -23,6 +23,11 @@ import static com.sap.piper.Prerequisites.checkScript
      * These parameters are appended to the npm command during execution.
      */
     'appUrls',
+    /**
+     * List of build descriptors and therefore modules to exclude from execution of the npm scripts.
+     * The elements of the list can either be a path to the build descriptor or a pattern.
+     */
+    'buildDescriptorExcludeList',
     /**
      * Script to be executed from package.json.
      */
@@ -93,14 +98,15 @@ void call(Map parameters = [:]) {
                 utils.unstashStageFiles(script, stageName)
                 try {
                     withCredentials(credentials) {
+                        List scriptOptions = ["--launchUrl=${appUrl.url}"]
                         if (appUrl.parameters) {
                             if (appUrl.parameters instanceof List) {
-                                npmExecuteScripts(script: script, parameters: npmParameters, install: false, virtualFrameBuffer: true, runScripts: [config.runScript], scriptOptions: ["--launchUrl=${appUrl.url}"] + appUrl.parameters)
+                                scriptOptions = scriptOptions + appUrl.parameters
                             } else {
                                 error "[${STEP_NAME}] The parameters property is not of type list. Please provide parameters as a list of strings."
                             }
                         }
-                        npmExecuteScripts(script: script, parameters: npmParameters, install: false, virtualFrameBuffer: true, runScripts: [config.runScript], scriptOptions: ["--launchUrl=${appUrl.url}"])
+                        npmExecuteScripts(script: script, parameters: npmParameters, install: false, virtualFrameBuffer: true, runScripts: [config.runScript], scriptOptions: scriptOptions, buildDescriptorExcludeList: config.buildDescriptorExcludeList)
                     }
 
                 } catch (Exception e) {