jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-02-11 13:53:53 +02:00
Code Issues Releases Activity

Add buildDescriptorExcludeList parameter to npmExecuteScripts step (#1800)

Browse Source 
This change adds a buildDescriptorExcludeList parameter to
npmExecuteScripts, to enable the exclusion of certain directories when
executing npm scripts. Previously, npmExecuteScripts could only execute
scripts in all packages.

Now it is possible to provide paths or patterns as elements of the
buildDescriptorExcludeList to exclude packages when executing npm scripts.
This commit is contained in:
Kevin Hudemann 2020-07-16 17:16:55 +02:00 committed by GitHub
parent 8af0540de2
commit 58e7e4be44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 132 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/npmExecuteLint.go
View File

@ -100,7 +100,7 @@ func runCiLint(npmExecutor npm.Executor, failOnError bool) error {
runScripts := []string{"ci-lint"}
runOptions := []string{"--silent"}
err := npmExecutor.RunScriptsInAllPackages(runScripts, runOptions, nil, false)
err := npmExecutor.RunScriptsInAllPackages(runScripts, runOptions, nil, false, nil)
if err != nil {
if failOnError {
return fmt.Errorf("ci-lint script execution failed with error: %w. This might be the result of severe linting findings, or some other issue while executing the script. Please examine the linting results in the UI, the cilint.xml file, if available, or the log above. ", err)

11
cmd/npmExecuteScripts.go
View File

@ -17,14 +17,17 @@ func npmExecuteScripts(config npmExecuteScriptsOptions, telemetryData *telemetry
}
func runNpmExecuteScripts(npmExecutor npm.Executor, config *npmExecuteScriptsOptions) error {
packageJSONFiles := npmExecutor.FindPackageJSONFiles()
if config.Install {
err := npmExecutor.InstallAllDependencies(packageJSONFiles)
packageJSONFiles, err := npmExecutor.FindPackageJSONFilesWithExcludes(config.BuildDescriptorExcludeList)
if err != nil {
return err
}
err = npmExecutor.InstallAllDependencies(packageJSONFiles)
if err != nil {
return err
}
}
return npmExecutor.RunScriptsInAllPackages(config.RunScripts, nil, config.ScriptOptions, config.VirtualFrameBuffer)
return npmExecutor.RunScriptsInAllPackages(config.RunScripts, nil, config.ScriptOptions, config.VirtualFrameBuffer, config.BuildDescriptorExcludeList)
}

22
cmd/npmExecuteScripts_generated.go
View File

@ -14,12 +14,13 @@ import (
)
type npmExecuteScriptsOptions struct {
Install bool `json:"install,omitempty"`
RunScripts []string `json:"runScripts,omitempty"`
DefaultNpmRegistry string `json:"defaultNpmRegistry,omitempty"`
SapNpmRegistry string `json:"sapNpmRegistry,omitempty"`
VirtualFrameBuffer bool `json:"virtualFrameBuffer,omitempty"`
ScriptOptions []string `json:"scriptOptions,omitempty"`
Install bool `json:"install,omitempty"`
RunScripts []string `json:"runScripts,omitempty"`
DefaultNpmRegistry string `json:"defaultNpmRegistry,omitempty"`
SapNpmRegistry string `json:"sapNpmRegistry,omitempty"`
VirtualFrameBuffer bool `json:"virtualFrameBuffer,omitempty"`
ScriptOptions []string `json:"scriptOptions,omitempty"`
BuildDescriptorExcludeList []string `json:"buildDescriptorExcludeList,omitempty"`
}
// NpmExecuteScriptsCommand Execute npm run scripts on all npm packages in a project
@ -83,6 +84,7 @@ func addNpmExecuteScriptsFlags(cmd *cobra.Command, stepConfig *npmExecuteScripts
cmd.Flags().StringVar(&stepConfig.SapNpmRegistry, "sapNpmRegistry", `https://npm.sap.com`, "The default npm registry URL to be used as the remote mirror for the SAP npm packages.")
cmd.Flags().BoolVar(&stepConfig.VirtualFrameBuffer, "virtualFrameBuffer", false, "(Linux only) Start a virtual frame buffer in the background. This allows you to run a web browser without the need for an X server. Note that xvfb needs to be installed in the execution environment.")
cmd.Flags().StringSliceVar(&stepConfig.ScriptOptions, "scriptOptions", []string{}, "Options are passed to all runScripts calls separated by a '--'. './piper npmExecuteScripts --runScripts ci-e2e --scriptOptions '--tag1' will correspond to 'npm run ci-e2e -- --tag1'")
cmd.Flags().StringSliceVar(&stepConfig.BuildDescriptorExcludeList, "buildDescriptorExcludeList", []string{`deployment/**`}, "List of build descriptors and therefore modules to exclude from execution of the npm scripts. The elements can either be a path to the build descriptor or a pattern.")
}
@ -144,6 +146,14 @@ func npmExecuteScriptsMetadata() config.StepData {
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "buildDescriptorExcludeList",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "[]string",
Mandatory: false,
Aliases: []config.Alias{},
},
},
},
},

27
cmd/npmExecuteScripts_test.go
View File

@ -32,6 +32,7 @@ type npmConfig struct {
runOptions []string
scriptOptions []string
virtualFrameBuffer bool
excludeList []string
}
// npmExecutorMock mocking struct
@ -46,13 +47,19 @@ func (n *npmExecutorMock) FindPackageJSONFiles() []string {
return packages
}
// FindPackageJSONFiles mock implementation
func (n *npmExecutorMock) FindPackageJSONFilesWithExcludes(excludeList []string) ([]string, error) {
packages, _ := n.utils.Glob("**/package.json")
return packages, nil
}
// FindPackageJSONFilesWithScript mock implementation
func (n *npmExecutorMock) FindPackageJSONFilesWithScript(packageJSONFiles []string, script string) ([]string, error) {
return packageJSONFiles, nil
}
// RunScriptsInAllPackages mock implementation
func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool) error {
func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error {
if len(runScripts) != len(n.config.runScripts) {
return fmt.Errorf("RunScriptsInAllPackages was called with a different list of runScripts than config.runScripts")
}
@ -63,7 +70,7 @@ func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOption
}
if len(scriptOptions) != len(n.config.scriptOptions) {
return fmt.Errorf("RunScriptsInAllPackages was called with a different list of runOptions than config.scriptOptions")
return fmt.Errorf("RunScriptsInAllPackages was called with a different list of scriptOptions than config.scriptOptions")
}
if len(runOptions) != len(n.config.runOptions) {
@ -74,6 +81,10 @@ func (n *npmExecutorMock) RunScriptsInAllPackages(runScripts []string, runOption
return fmt.Errorf("RunScriptsInAllPackages was called with a different value of virtualFrameBuffer than config.virtualFrameBuffer")
}
if len(excludeList) != len(n.config.excludeList) {
return fmt.Errorf("RunScriptsInAllPackages was called with a different value of excludeList than config.excludeList")
}
return nil
}
@ -101,6 +112,18 @@ func (n *npmExecutorMock) SetNpmRegistries() error {
}
func TestNpmExecuteScripts(t *testing.T) {
t.Run("Call with excludeList", func(t *testing.T) {
config := npmExecuteScriptsOptions{Install: true, RunScripts: []string{"ci-build", "ci-test"}, BuildDescriptorExcludeList: []string{"**/path/**"}}
utils := newNpmMockUtilsBundle()
utils.AddFile("package.json", []byte("{\"name\": \"Test\" }"))
utils.AddFile("src/package.json", []byte("{\"name\": \"Test\" }"))
npmExecutor := npmExecutorMock{utils: utils, config: npmConfig{install: config.Install, runScripts: config.RunScripts, excludeList: config.BuildDescriptorExcludeList}}
err := runNpmExecuteScripts(&npmExecutor, &config)
assert.NoError(t, err)
})
t.Run("Call with scriptOptions", func(t *testing.T) {
config := npmExecuteScriptsOptions{Install: true, RunScripts: []string{"ci-build", "ci-test"}, ScriptOptions: []string{"--run"}}
utils := newNpmMockUtilsBundle()

42
pkg/npm/npm.go
View File

@ -7,8 +7,8 @@ import (
"github.com/SAP/jenkins-library/pkg/command"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/bmatcuk/doublestar"
"io"
"os"
"path/filepath"
"strings"
)
@ -22,8 +22,9 @@ type Execute struct {
// Executor interface to enable mocking for testing
type Executor interface {
FindPackageJSONFiles() []string
FindPackageJSONFilesWithExcludes(excludeList []string) ([]string, error)
FindPackageJSONFilesWithScript(packageJSONFiles []string, script string) ([]string, error)
RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool) error
RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error
InstallAllDependencies(packageJSONFiles []string) error
SetNpmRegistries() error
}
@ -128,8 +129,12 @@ func registryRequiresConfiguration(preConfiguredRegistry, url string) bool {
}
// RunScriptsInAllPackages runs all scripts defined in ExecuteOptions.RunScripts
func (exec *Execute) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool) error {
packageJSONFiles := exec.FindPackageJSONFiles()
func (exec *Execute) RunScriptsInAllPackages(runScripts []string, runOptions []string, scriptOptions []string, virtualFrameBuffer bool, excludeList []string) error {
packageJSONFiles, err := exec.FindPackageJSONFilesWithExcludes(excludeList)
if err != nil {
return err
}
execRunner := exec.Utils.GetExecRunner()
if virtualFrameBuffer {
@ -205,25 +210,42 @@ func (exec *Execute) executeScript(packageJSON string, script string, runOptions
return nil
}
// FindPackageJSONFiles returns a list of all package.json fileUtils of the project excluding node_modules and gen/ directories
// FindPackageJSONFiles returns a list of all package.json files of the project excluding node_modules and gen/ directories
func (exec *Execute) FindPackageJSONFiles() []string {
packageJSONFiles, _ := exec.FindPackageJSONFilesWithExcludes([]string{})
return packageJSONFiles
}
// FindPackageJSONFilesWithExcludes returns a list of all package.json files of the project excluding node_modules, gen/ and directories/patterns defined by excludeList
func (exec *Execute) FindPackageJSONFilesWithExcludes(excludeList []string) ([]string, error) {
unfilteredListOfPackageJSONFiles, _ := exec.Utils.Glob("**/package.json")
nodeModulesExclude := "**/node_modules/**"
genExclude := "**/gen/**"
excludeList = append(excludeList, nodeModulesExclude, genExclude)
var packageJSONFiles []string
for _, file := range unfilteredListOfPackageJSONFiles {
if strings.Contains(file, "node_modules") {
continue
excludePackage := false
for _, exclude := range excludeList {
matched, err := doublestar.PathMatch(exclude, file)
if err != nil {
return nil, fmt.Errorf("failed to match file %s to pattern %s: %w", file, exclude, err)
}
if matched {
excludePackage = true
break
}
}
if strings.HasPrefix(file, "gen"+string(os.PathSeparator)) || strings.Contains(file, string(os.PathSeparator)+"gen"+string(os.PathSeparator)) {
if excludePackage {
continue
}
packageJSONFiles = append(packageJSONFiles, file)
log.Entry().Info("Discovered package.json file " + file)
}
return packageJSONFiles
return packageJSONFiles, nil
}
// FindPackageJSONFilesWithScript returns a list of package.json fileUtils that contain the script

37
pkg/npm/npm_test.go
View File

@ -36,13 +36,16 @@ func TestNpm(t *testing.T) {
packageJSONFiles := exec.FindPackageJSONFiles()
assert.Equal(t, []string{"package.json"}, packageJSONFiles)
})
t.Run("find package.json files with two package.json and filtered package.json", func(t *testing.T) {
t.Run("find package.json files with two package.json and default filter", func(t *testing.T) {
utils := newNpmMockUtilsBundle()
utils.AddFile("package.json", []byte("{}"))
utils.AddFile(filepath.Join("src", "package.json"), []byte("{}"))
utils.AddFile(filepath.Join("src", "package.json"), []byte("{}")) // should NOT be filtered out
utils.AddFile(filepath.Join("node_modules", "package.json"), []byte("{}")) // is filtered out
utils.AddFile(filepath.Join("gen", "package.json"), []byte("{}")) // is filtered out
options := ExecutorOptions{}
exec := &Execute{
@ -55,6 +58,32 @@ func TestNpm(t *testing.T) {
assert.Equal(t, []string{"package.json", filepath.Join("src", "package.json")}, packageJSONFiles)
})
t.Run("find package.json files with two package.json and excludes", func(t *testing.T) {
utils := newNpmMockUtilsBundle()
utils.AddFile("package.json", []byte("{}"))
utils.AddFile(filepath.Join("src", "package.json"), []byte("{}")) // should NOT be filtered out
utils.AddFile(filepath.Join("notfiltered", "package.json"), []byte("{}")) // should NOT be filtered out
utils.AddFile(filepath.Join("Path", "To", "filter", "package.json"), []byte("{}")) // should NOT be filtered out
utils.AddFile(filepath.Join("node_modules", "package.json"), []byte("{}")) // is filtered out
utils.AddFile(filepath.Join("gen", "package.json"), []byte("{}")) // is filtered out
utils.AddFile(filepath.Join("filter", "package.json"), []byte("{}")) // is filtered out
utils.AddFile(filepath.Join("filterPath", "package.json"), []byte("{}")) // is filtered out
utils.AddFile(filepath.Join("filter", "Path", "To", "package.json"), []byte("{}")) // is filtered out
options := ExecutorOptions{}
exec := &Execute{
Utils: &utils,
Options: options,
}
packageJSONFiles, err := exec.FindPackageJSONFilesWithExcludes([]string{"filter/**", "filterPath/package.json"})
if assert.NoError(t, err) {
assert.Equal(t, []string{filepath.Join("Path", "To", "filter", "package.json"), filepath.Join("notfiltered", "package.json"), "package.json", filepath.Join("src", "package.json")}, packageJSONFiles)
}
})
t.Run("find package.json files with script", func(t *testing.T) {
utils := newNpmMockUtilsBundle()
utils.AddFile("package.json", []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }"))
@ -231,7 +260,7 @@ func TestNpm(t *testing.T) {
Utils: &utils,
Options: options,
}
err := exec.RunScriptsInAllPackages(runScripts, nil, nil, false)
err := exec.RunScriptsInAllPackages(runScripts, nil, nil, false, nil)
if assert.NoError(t, err) {
if assert.Equal(t, 6, len(utils.execRunner.Calls)) {
@ -295,7 +324,7 @@ func TestNpm(t *testing.T) {
Utils: &utils,
Options: options,
}
err := exec.RunScriptsInAllPackages([]string{"foo"}, nil, nil, true)
err := exec.RunScriptsInAllPackages([]string{"foo"}, nil, nil, true, nil)
assert.Contains(t, utils.execRunner.Env, "DISPLAY=:99")
assert.NoError(t, err)

8
resources/metadata/npmExecuteScripts.yaml
View File

@ -58,6 +58,14 @@ spec:
- PARAMETERS
- STAGES
- STEPS
- name: buildDescriptorExcludeList
type: '[]string'
description: List of build descriptors and therefore modules to exclude from execution of the npm scripts. The elements can either be a path to the build descriptor or a pattern.
scope:
- PARAMETERS
- STAGES
- STEPS
default: ["deployment/**"]
containers:
- name: node
image: node:12-buster-slim

2
test/groovy/NpmExecuteEndToEndTestsTest.groovy
View File

@ -223,6 +223,7 @@ class NpmExecuteEndToEndTestsTest extends BasePiperTest {
stepRule.step.npmExecuteEndToEndTests(
script: nullScript,
stage: "myStage",
buildDescriptorExcludeList: ["path/to/package.json"],
runScript: "ci-e2e"
)
@ -232,6 +233,7 @@ class NpmExecuteEndToEndTestsTest extends BasePiperTest {
assert npmExecuteScriptsRule.hasParameter('virtualFrameBuffer', true)
assert npmExecuteScriptsRule.hasParameter('runScripts', ["ci-e2e"])
assert npmExecuteScriptsRule.hasParameter('scriptOptions', ["--launchUrl=${appUrl.url}"] + appUrl.parameters)
assert npmExecuteScriptsRule.hasParameter('buildDescriptorExcludeList', ["path/to/package.json"])
}
@Test

10
vars/npmExecuteEndToEndTests.groovy
View File

@ -23,6 +23,11 @@ import static com.sap.piper.Prerequisites.checkScript
* These parameters are appended to the npm command during execution.
*/
'appUrls',
/**
* List of build descriptors and therefore modules to exclude from execution of the npm scripts.
* The elements of the list can either be a path to the build descriptor or a pattern.
*/
'buildDescriptorExcludeList',
/**
* Script to be executed from package.json.
*/
@ -93,14 +98,15 @@ void call(Map parameters = [:]) {
utils.unstashStageFiles(script, stageName)
try {
withCredentials(credentials) {
List scriptOptions = ["--launchUrl=${appUrl.url}"]
if (appUrl.parameters) {
if (appUrl.parameters instanceof List) {
npmExecuteScripts(script: script, parameters: npmParameters, install: false, virtualFrameBuffer: true, runScripts: [config.runScript], scriptOptions: ["--launchUrl=${appUrl.url}"] + appUrl.parameters)
scriptOptions = scriptOptions + appUrl.parameters
} else {
error "[${STEP_NAME}] The parameters property is not of type list. Please provide parameters as a list of strings."
}
}
npmExecuteScripts(script: script, parameters: npmParameters, install: false, virtualFrameBuffer: true, runScripts: [config.runScript], scriptOptions: ["--launchUrl=${appUrl.url}"])
npmExecuteScripts(script: script, parameters: npmParameters, install: false, virtualFrameBuffer: true, runScripts: [config.runScript], scriptOptions: scriptOptions, buildDescriptorExcludeList: config.buildDescriptorExcludeList)
}
} catch (Exception e) {