WhiteSource: Force update of project when "checkPolicies" failed (#2401)

* Force update of WS project
* Refactor file filtering to avoid duplicated code

pkg/npm/npm.go

@@ -7,7 +7,6 @@ import (
 	"github.com/SAP/jenkins-library/pkg/command"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
-	"github.com/bmatcuk/doublestar"
 	"io"
 	"path/filepath"
 	"strings"
@@ -220,25 +219,12 @@ func (exec *Execute) FindPackageJSONFilesWithExcludes(excludeList []string) ([]s
 	genExclude := "**/gen/**"
 	excludeList = append(excludeList, nodeModulesExclude, genExclude)
 
-	var packageJSONFiles []string
+	packageJSONFiles, err := piperutils.ExcludeFiles(unfilteredListOfPackageJSONFiles, excludeList)
+	if err != nil {
+		return nil, err
+	}
 
-	for _, file := range unfilteredListOfPackageJSONFiles {
-		excludePackage := false
-		for _, exclude := range excludeList {
-			matched, err := doublestar.PathMatch(exclude, file)
-			if err != nil {
-				return nil, fmt.Errorf("failed to match file %s to pattern %s: %w", file, exclude, err)
-			}
-			if matched {
-				excludePackage = true
-				break
-			}
-		}
-		if excludePackage {
-			continue
-		}
-
-		packageJSONFiles = append(packageJSONFiles, file)
+	for _, file := range packageJSONFiles {
 		log.Entry().Info("Discovered package.json file " + file)
 	}
 	return packageJSONFiles, nil

pkg/piperutils/FileUtils.go

@@ -228,6 +228,34 @@ func (f Files) Glob(pattern string) (matches []string, err error) {
 	return doublestar.Glob(pattern)
 }
 
+// ExcludeFiles returns a slice of files, which contains only the sub-set of files that matched none
+// of the glob patterns in the provided excludes list.
+func ExcludeFiles(files, excludes []string) ([]string, error) {
+	if len(excludes) == 0 {
+		return files, nil
+	}
+
+	var filteredFiles []string
+	for _, file := range files {
+		includeFile := true
+		for _, exclude := range excludes {
+			matched, err := doublestar.PathMatch(exclude, file)
+			if err != nil {
+				return nil, fmt.Errorf("failed to match file %s to pattern %s: %w", file, exclude, err)
+			}
+			if matched {
+				includeFile = false
+				break
+			}
+		}
+		if includeFile {
+			filteredFiles = append(filteredFiles, file)
+		}
+	}
+
+	return filteredFiles, nil
+}
+
 // Getwd is a wrapper for os.Getwd().
 func (f Files) Getwd() (string, error) {
 	return os.Getwd()

pkg/piperutils/fileUtils_test.go

@@ -96,3 +96,55 @@ func runInTempDir(t *testing.T, nameOfRun, tempDirPattern string, run func(t *te
 
 	t.Run(nameOfRun, run)
 }
+
+func TestExcludeFiles(t *testing.T) {
+	t.Parallel()
+	t.Run("nil slices", func(t *testing.T) {
+		t.Parallel()
+		filtered, err := ExcludeFiles(nil, nil)
+		assert.NoError(t, err)
+		assert.Len(t, filtered, 0)
+	})
+	t.Run("empty excludes", func(t *testing.T) {
+		t.Parallel()
+		files := []string{"file"}
+		filtered, err := ExcludeFiles(files, nil)
+		assert.NoError(t, err)
+		assert.Equal(t, files, filtered)
+	})
+	t.Run("direct match", func(t *testing.T) {
+		t.Parallel()
+		files := []string{"file"}
+		filtered, err := ExcludeFiles(files, files)
+		assert.NoError(t, err)
+		assert.Len(t, filtered, 0)
+	})
+	t.Run("two direct matches", func(t *testing.T) {
+		t.Parallel()
+		files := []string{"a", "b"}
+		filtered, err := ExcludeFiles(files, files)
+		assert.NoError(t, err)
+		assert.Len(t, filtered, 0)
+	})
+	t.Run("one direct exclude matches", func(t *testing.T) {
+		t.Parallel()
+		files := []string{"a", "b"}
+		filtered, err := ExcludeFiles(files, []string{"b"})
+		assert.NoError(t, err)
+		assert.Equal(t, []string{"a"}, filtered)
+	})
+	t.Run("no glob matches", func(t *testing.T) {
+		t.Parallel()
+		files := []string{"a", "b"}
+		filtered, err := ExcludeFiles(files, []string{"*/a", "b/*"})
+		assert.NoError(t, err)
+		assert.Equal(t, []string{"a", "b"}, filtered)
+	})
+	t.Run("two globs match", func(t *testing.T) {
+		t.Parallel()
+		files := []string{"path/to/a", "b"}
+		filtered, err := ExcludeFiles(files, []string{"**/a", "**/b"})
+		assert.NoError(t, err)
+		assert.Len(t, filtered, 0)
+	})
+}

pkg/whitesource/scanNPM.go

@@ -49,6 +49,9 @@ func (s *Scan) writeWhitesourceConfigJSON(config *ScanOptions, utils Utils, devD
 	npmConfig["apiKey"] = config.OrgToken
 	npmConfig["userKey"] = config.UserToken
 	setValueAndLogChange(npmConfig, "checkPolicies", true)
+	// When checkPolicies detects any violations, it will by default not update the WS project in the backend.
+	// Therefore we also need "forceUpdate".
+	setValueAndLogChange(npmConfig, "forceUpdate", true)
 	setValueAndLogChange(npmConfig, "productName", config.ProductName)
 	setValueAndLogChange(npmConfig, "productVer", s.ProductVersion)
 	setValueOmitIfPresent(npmConfig, "productToken", "projectToken", config.ProductToken)

pkg/whitesource/scanNPM_test.go

@@ -49,6 +49,48 @@ func TestExecuteScanNPM(t *testing.T) {
 		assert.True(t, utilsMock.HasWrittenFile(whiteSourceConfig))
 		assert.True(t, utilsMock.HasRemovedFile(whiteSourceConfig))
 	})
+	t.Run("happy path with excluded modules", func(t *testing.T) {
+		// init
+		utilsMock := NewScanUtilsMock()
+		utilsMock.AddFile("package.json", []byte(`{"name":"my-module-name"}`))
+		utilsMock.AddFile("sub/package.json", []byte(`{"name":"my-sub-module-name"}`))
+		utilsMock.AddFile("deep/sub/package.json", []byte(`{"name":"my-deep-sub-module-name"}`))
+
+		config := ScanOptions{
+			ScanType:                   "npm",
+			OrgToken:                   "org-token",
+			UserToken:                  "user-token",
+			ProductName:                "mock-product",
+			ProjectName:                "mock-project",
+			BuildDescriptorExcludeList: []string{"unrelated/pom.xml", "sub/package.json", "deep/sub/package.json"},
+		}
+
+		scan := newTestScan(&config)
+		// test
+		err := scan.ExecuteNpmScan(&config, utilsMock)
+		// assert
+		require.NoError(t, err)
+		expectedCalls := []mock.ExecCall{
+			{
+				Exec: "npm",
+				Params: []string{
+					"ls",
+				},
+			},
+			{
+				Exec: "npx",
+				Params: []string{
+					"whitesource",
+					"run",
+				},
+			},
+		}
+		assert.Equal(t, expectedCalls, utilsMock.Calls)
+		assert.True(t, utilsMock.HasWrittenFile(whiteSourceConfig))
+		assert.True(t, utilsMock.HasRemovedFile(whiteSourceConfig))
+		assert.False(t, utilsMock.HasWrittenFile(filepath.Join("sub", whiteSourceConfig)))
+		assert.False(t, utilsMock.HasWrittenFile(filepath.Join("deep", "sub", whiteSourceConfig)))
+	})
 	t.Run("no NPM modules", func(t *testing.T) {
 		// init
 		utilsMock := NewScanUtilsMock()
@@ -106,6 +148,7 @@ func TestWriteWhitesourceConfigJSON(t *testing.T) {
 	expected["apiKey"] = "org-token"
 	expected["userKey"] = "user-token"
 	expected["checkPolicies"] = true
+	expected["forceUpdate"] = true
 	expected["productName"] = "mock-product"
 	expected["projectName"] = "mock-project"
 	expected["productToken"] = "mock-product-token"

pkg/whitesource/utilsMock.go

@@ -4,6 +4,7 @@ package whitesource
 
 import (
 	"github.com/SAP/jenkins-library/pkg/mock"
+	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"net/http"
 	"os"
 )
@@ -42,9 +43,9 @@ func (m *ScanUtilsMock) RemoveAll(_ string) error {
 }
 
 // FindPackageJSONFiles mimics npm.FindPackageJSONFiles() based on the FilesMock setup.
-func (m *ScanUtilsMock) FindPackageJSONFiles(_ *ScanOptions) ([]string, error) {
-	matches, _ := m.Glob("**/package.json")
-	return matches, nil
+func (m *ScanUtilsMock) FindPackageJSONFiles(options *ScanOptions) ([]string, error) {
+	unfilteredMatches, _ := m.Glob("**/package.json")
+	return piperutils.ExcludeFiles(unfilteredMatches, options.BuildDescriptorExcludeList)
 }
 
 // InstallAllNPMDependencies mimics npm.InstallAllNPMDependencies() and records the "npm install".