npmExecuteScripts: allow passing a list of build descriptors to execute scripts for (#2312)

This change extends the npmExecuteScripts step to support execution of
npm scripts for specific modules. Previously, it was not possible to
execute npm scripts only for specific modules. Now, if the parameter
buildDesriptorList is set the scripts defined by the runScripts
parameter will be executed for the modules defined by
buildDescriptorList. Note, in this case the buildDescriptorExcludeList
will be ignored.

cmd/npmExecuteLint.go

@@ -100,7 +100,7 @@ func runCiLint(npmExecutor npm.Executor, failOnError bool) error {
 	runScripts := []string{"ci-lint"}
 	runOptions := []string{"--silent"}
 
-	err := npmExecutor.RunScriptsInAllPackages(runScripts, runOptions, nil, false, nil)
+	err := npmExecutor.RunScriptsInAllPackages(runScripts, runOptions, nil, false, nil, nil)
 	if err != nil {
 		if failOnError {
 			return fmt.Errorf("ci-lint script execution failed with error: %w. This might be the result of severe linting findings, or some other issue while executing the script. Please examine the linting results in the UI, the cilint.xml file, if available, or the log above. ", err)

cmd/npmExecuteScripts.go

@@ -29,5 +29,5 @@ func runNpmExecuteScripts(npmExecutor npm.Executor, config *npmExecuteScriptsOpt
 		}
 	}
 
-	return npmExecutor.RunScriptsInAllPackages(config.RunScripts, nil, config.ScriptOptions, config.VirtualFrameBuffer, config.BuildDescriptorExcludeList)
+	return npmExecutor.RunScriptsInAllPackages(config.RunScripts, nil, config.ScriptOptions, config.VirtualFrameBuffer, config.BuildDescriptorExcludeList, config.BuildDescriptorList)
 }

cmd/npmExecuteScripts_generated.go

@@ -20,6 +20,7 @@ type npmExecuteScriptsOptions struct {
 	VirtualFrameBuffer         bool     `json:"virtualFrameBuffer,omitempty"`
 	ScriptOptions              []string `json:"scriptOptions,omitempty"`
 	BuildDescriptorExcludeList []string `json:"buildDescriptorExcludeList,omitempty"`
+	BuildDescriptorList        []string `json:"buildDescriptorList,omitempty"`
 }
 
 // NpmExecuteScriptsCommand Execute npm run scripts on all npm packages in a project
@@ -85,6 +86,7 @@ func addNpmExecuteScriptsFlags(cmd *cobra.Command, stepConfig *npmExecuteScripts
 	cmd.Flags().BoolVar(&stepConfig.VirtualFrameBuffer, "virtualFrameBuffer", false, "(Linux only) Start a virtual frame buffer in the background. This allows you to run a web browser without the need for an X server. Note that xvfb needs to be installed in the execution environment.")
 	cmd.Flags().StringSliceVar(&stepConfig.ScriptOptions, "scriptOptions", []string{}, "Options are passed to all runScripts calls separated by a '--'. './piper npmExecuteScripts --runScripts ci-e2e --scriptOptions '--tag1' will correspond to 'npm run ci-e2e -- --tag1'")
 	cmd.Flags().StringSliceVar(&stepConfig.BuildDescriptorExcludeList, "buildDescriptorExcludeList", []string{`deployment/**`}, "List of build descriptors and therefore modules to exclude from execution of the npm scripts. The elements can either be a path to the build descriptor or a pattern.")
+	cmd.Flags().StringSliceVar(&stepConfig.BuildDescriptorList, "buildDescriptorList", []string{}, "List of build descriptors and therefore modules for execution of the npm scripts. The elements have to be paths to the build descriptors. **If set, buildDescriptorExcludeList will be ignored.**")
 
 }
 
@@ -146,6 +148,14 @@ func npmExecuteScriptsMetadata() config.StepData {
 						Mandatory:   false,
 						Aliases:     []config.Alias{},
 					},
+					{
+						Name:        "buildDescriptorList",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "[]string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+					},
 				},
 			},
 		},

cmd/npmExecuteScripts_test.go

@@ -33,6 +33,7 @@ type npmConfig struct {
 	scriptOptions      []string
 	virtualFrameBuffer bool
 	excludeList        []string
+	packagesList       []string
 }
 
 // npmExecutorMock mocking struct
@@ -59,7 +60,7 @@ func (n *npmExecutorMock) FindPackageJSONFilesWithScript(packageJSONFiles []stri
 }
 
 // RunScriptsInAllPackages mock implementation
-func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error {
+func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string, packagesList []string) error {
 	if len(runScripts) != len(n.config.runScripts) {
 		return fmt.Errorf("RunScriptsInAllPackages was called with a different list of runScripts than config.runScripts")
 	}
@@ -85,6 +86,10 @@ func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOption
 		return fmt.Errorf("RunScriptsInAllPackages was called with a different value of excludeList than config.excludeList")
 	}
 
+	if len(packagesList) != len(n.config.packagesList) {
+		return fmt.Errorf("RunScriptsInAllPackages was called with a different value of packagesList than config.packagesList")
+	}
+
 	return nil
 }
 
@@ -112,6 +117,18 @@ func (n *npmExecutorMock) SetNpmRegistries() error {
 }
 
 func TestNpmExecuteScripts(t *testing.T) {
+	t.Run("Call with packagesList", func(t *testing.T) {
+		config := npmExecuteScriptsOptions{Install: true, RunScripts: []string{"ci-build", "ci-test"}, BuildDescriptorList: []string{"src/package.json"}}
+		utils := newNpmMockUtilsBundle()
+		utils.AddFile("package.json", []byte("{\"name\": \"Test\" }"))
+		utils.AddFile("src/package.json", []byte("{\"name\": \"Test\" }"))
+
+		npmExecutor := npmExecutorMock{utils: utils, config: npmConfig{install: config.Install, runScripts: config.RunScripts, packagesList: config.BuildDescriptorList}}
+		err := runNpmExecuteScripts(&npmExecutor, &config)
+
+		assert.NoError(t, err)
+	})
+
 	t.Run("Call with excludeList", func(t *testing.T) {
 		config := npmExecuteScriptsOptions{Install: true, RunScripts: []string{"ci-build", "ci-test"}, BuildDescriptorExcludeList: []string{"**/path/**"}}
 		utils := newNpmMockUtilsBundle()

pkg/npm/npm.go

@@ -24,7 +24,7 @@ type Executor interface {
 	FindPackageJSONFiles() []string
 	FindPackageJSONFilesWithExcludes(excludeList []string) ([]string, error)
 	FindPackageJSONFilesWithScript(packageJSONFiles []string, script string) ([]string, error)
-	RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error
+	RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string, packagesList []string) error
 	InstallAllDependencies(packageJSONFiles []string) error
 	SetNpmRegistries() error
 }
@@ -118,10 +118,17 @@ func registryRequiresConfiguration(preConfiguredRegistry, url string) bool {
 }
 
 // RunScriptsInAllPackages runs all scripts defined in ExecuteOptions.RunScripts
-func (exec *Execute) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error {
+func (exec *Execute) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string, packagesList []string) error {
-	packageJSONFiles, err := exec.FindPackageJSONFilesWithExcludes(excludeList)
+	var packageJSONFiles []string
-	if err != nil {
+	var err error
-		return err
+
+	if len(packagesList) > 0 {
+		packageJSONFiles = packagesList
+	} else {
+		packageJSONFiles, err = exec.FindPackageJSONFilesWithExcludes(excludeList)
+		if err != nil {
+			return err
+		}
 	}
 
 	execRunner := exec.Utils.GetExecRunner()
@@ -269,7 +276,15 @@ func (exec *Execute) FindPackageJSONFilesWithScript(packageJSONFiles []string, s
 // InstallAllDependencies executes npm or yarn Install for all package.json fileUtils defined in packageJSONFiles
 func (exec *Execute) InstallAllDependencies(packageJSONFiles []string) error {
 	for _, packageJSON := range packageJSONFiles {
-		err := exec.install(packageJSON)
+		fileExists, err := exec.Utils.FileExists(packageJSON)
+		if err != nil {
+			return fmt.Errorf("cannot check if '%s' exists: %w", packageJSON, err)
+		}
+		if !fileExists {
+			return fmt.Errorf("package.json file '%s' not found: %w", packageJSON, err)
+		}
+
+		err = exec.install(packageJSON)
 		if err != nil {
 			return err
 		}

pkg/npm/npm_test.go

@@ -259,7 +259,7 @@ func TestNpm(t *testing.T) {
 			Utils:   &utils,
 			Options: options,
 		}
-		err := exec.RunScriptsInAllPackages(runScripts, nil, nil, false, nil)
+		err := exec.RunScriptsInAllPackages(runScripts, nil, nil, false, nil, nil)
 
 		if assert.NoError(t, err) {
 			if assert.Equal(t, 4, len(utils.execRunner.Calls)) {
@@ -269,6 +269,28 @@ func TestNpm(t *testing.T) {
 		}
 	})
 
+	t.Run("check Execute all scripts with buildDescriptorList", func(t *testing.T) {
+		utils := newNpmMockUtilsBundle()
+		utils.AddFile("package.json", []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }"))                        // is filtered out
+		utils.AddFile(filepath.Join("src", "package.json"), []byte("{\"scripts\": { \"ci-build\": \"exit 0\" } }")) // should NOT be filtered out
+
+		options := ExecutorOptions{}
+		runScripts := []string{"ci-lint", "ci-build"}
+		buildDescriptorList := []string{filepath.Join("src", "package.json")}
+
+		exec := &Execute{
+			Utils:   &utils,
+			Options: options,
+		}
+		err := exec.RunScriptsInAllPackages(runScripts, nil, nil, false, nil, buildDescriptorList)
+
+		if assert.NoError(t, err) {
+			if assert.Equal(t, 2, len(utils.execRunner.Calls)) {
+				assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run", "ci-build"}}, utils.execRunner.Calls[1])
+			}
+		}
+	})
+
 	t.Run("check set npm registry", func(t *testing.T) {
 		utils := newNpmMockUtilsBundle()
 		utils.AddFile("package.json", []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }"))
@@ -301,7 +323,7 @@ func TestNpm(t *testing.T) {
 			Utils:   &utils,
 			Options: options,
 		}
-		err := exec.RunScriptsInAllPackages([]string{"foo"}, nil, nil, true, nil)
+		err := exec.RunScriptsInAllPackages([]string{"foo"}, nil, nil, true, nil, nil)
 
 		assert.Contains(t, utils.execRunner.Env, "DISPLAY=:99")
 		assert.NoError(t, err)

resources/metadata/npmExecuteScripts.yaml

@@ -58,6 +58,13 @@ spec:
           - STAGES
           - STEPS
         default: ["deployment/**"]
+      - name: buildDescriptorList
+        type: "[]string"
+        description: List of build descriptors and therefore modules for execution of the npm scripts. The elements have to be paths to the build descriptors. **If set, buildDescriptorExcludeList will be ignored.**
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
   containers:
     - name: node
       image: node:12-buster-slim