feat(cnbBuild) Introducing preserveFiles config to copy back files (#3562)

Co-authored-by: Ralf Pannemans <ralf.pannemans@sap.com>

cmd/cnbBuild.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strings"
 
 	"github.com/SAP/jenkins-library/pkg/certutils"
 	"github.com/SAP/jenkins-library/pkg/cnbutils"
@@ -147,31 +146,6 @@ func cnbBuild(config cnbBuildOptions, telemetryData *telemetry.CustomData, commo
 	}
 }
 
-func isIgnored(find string, include, exclude *ignore.GitIgnore) bool {
-	if exclude != nil {
-		filtered := exclude.MatchesPath(find)
-
-		if filtered {
-			log.Entry().Debugf("%s matches exclude pattern, ignoring", find)
-			return true
-		}
-	}
-
-	if include != nil {
-		filtered := !include.MatchesPath(find)
-
-		if filtered {
-			log.Entry().Debugf("%s doesn't match include pattern, ignoring", find)
-			return true
-		} else {
-			log.Entry().Debugf("%s matches include pattern", find)
-			return false
-		}
-	}
-
-	return false
-}
-
 func isBuilder(utils cnbutils.BuildUtils) error {
 	exists, err := utils.FileExists(creatorPath)
 	if err != nil {
@@ -215,62 +189,6 @@ func cleanDir(dir string, utils cnbutils.BuildUtils) error {
 	return nil
 }
 
-func copyFile(source, target string, utils cnbutils.BuildUtils) error {
-	targetDir := filepath.Dir(target)
-
-	exists, err := utils.DirExists(targetDir)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		log.Entry().Debugf("Creating directory %s", targetDir)
-		err = utils.MkdirAll(targetDir, os.ModePerm)
-		if err != nil {
-			return err
-		}
-	}
-
-	_, err = utils.Copy(source, target)
-	return err
-}
-
-func copyProject(source, target string, include, exclude *ignore.GitIgnore, utils cnbutils.BuildUtils) error {
-	sourceFiles, _ := utils.Glob(path.Join(source, "**"))
-	for _, sourceFile := range sourceFiles {
-		relPath, err := filepath.Rel(source, sourceFile)
-		if err != nil {
-			log.SetErrorCategory(log.ErrorBuild)
-			return errors.Wrapf(err, "Calculating relative path for '%s' failed", sourceFile)
-		}
-		if !isIgnored(relPath, include, exclude) {
-			target := path.Join(target, strings.ReplaceAll(sourceFile, source, ""))
-			dir, err := utils.DirExists(sourceFile)
-			if err != nil {
-				log.SetErrorCategory(log.ErrorBuild)
-				return errors.Wrapf(err, "Checking file info '%s' failed", target)
-			}
-
-			if dir {
-				err = utils.MkdirAll(target, os.ModePerm)
-				if err != nil {
-					log.SetErrorCategory(log.ErrorBuild)
-					return errors.Wrapf(err, "Creating directory '%s' failed", target)
-				}
-			} else {
-				log.Entry().Debugf("Copying '%s' to '%s'", sourceFile, target)
-				err = copyFile(sourceFile, target, utils)
-				if err != nil {
-					log.SetErrorCategory(log.ErrorBuild)
-					return errors.Wrapf(err, "Copying '%s' to '%s' failed", sourceFile, target)
-				}
-			}
-
-		}
-	}
-	return nil
-}
-
 func extractZip(source, target string) error {
 	if isZip(source) {
 		log.Entry().Infof("Extracting archive '%s' to '%s'", source, target)
@@ -553,7 +471,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, t
 	}
 
 	if pathType != pathEnumArchive {
-		err = copyProject(source, target, include, exclude, utils)
+		err = cnbutils.CopyProject(source, target, include, exclude, utils)
 		if err != nil {
 			log.SetErrorCategory(log.ErrorBuild)
 			return errors.Wrapf(err, "Copying  '%s' into '%s' failed", source, target)
@@ -636,5 +554,17 @@ func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, t
 		return errors.Wrapf(err, "execution of '%s' failed", creatorArgs)
 	}
 
+	if len(config.PreserveFiles) > 0 {
+		if pathType != pathEnumArchive {
+			err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils)
+			if err != nil {
+				log.SetErrorCategory(log.ErrorBuild)
+				return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles)
+			}
+		} else {
+			log.Entry().Warnf("skipping preserving files because the source '%s' is an archive", source)
+		}
+	}
+
 	return nil
 }

cmd/cnbBuild_generated.go

@@ -30,6 +30,7 @@ type cnbBuildOptions struct {
 	AdditionalTags            []string                 `json:"additionalTags,omitempty"`
 	Bindings                  map[string]interface{}   `json:"bindings,omitempty"`
 	MultipleImages            []map[string]interface{} `json:"multipleImages,omitempty"`
+	PreserveFiles             []string                 `json:"preserveFiles,omitempty"`
 }
 
 type cnbBuildCommonPipelineEnvironment struct {
@@ -165,11 +166,13 @@ func addCnbBuildFlags(cmd *cobra.Command, stepConfig *cnbBuildOptions) {
 	cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of '$HOSTNAME/$REPO[:$TAG]'.")
 
 	cmd.Flags().StringVar(&stepConfig.Path, "path", os.Getenv("PIPER_path"), "The path should either point to a directory with your sources or an artifact in zip format.\nThis property determines the input to the buildpack.\n")
-	cmd.Flags().StringVar(&stepConfig.ProjectDescriptor, "projectDescriptor", `project.toml`, "Relative path to the project.toml file.\nSee [buildpacks.io](https://buildpacks.io/docs/reference/config/project-descriptor/) for the reference.\nParameters passed to the cnbBuild step will take precedence over the parameters set in the project.toml file, except the `env` block.\nEnvironment variables declared in a project descriptor file, will be merged with the `buildEnvVars` property, with the `buildEnvVars` having a precedence.\n\nNote: The project descriptor path should be relative to what is set in the [path](#path) property. If the `path` property is pointing to a zip archive (e.g. jar file), project descriptor path will be relative to the root of the workspace.\nNote: Inline buildpacks (see [specification](https://buildpacks.io/docs/reference/config/project-descriptor/#build-_table-optional_)) are not supported yet.\n")
+	cmd.Flags().StringVar(&stepConfig.ProjectDescriptor, "projectDescriptor", `project.toml`, "Relative path to the project.toml file.\nSee [buildpacks.io](https://buildpacks.io/docs/reference/config/project-descriptor/) for the reference.\nParameters passed to the cnbBuild step will take precedence over the parameters set in the project.toml file, except the `env` block.\nEnvironment variables declared in a project descriptor file, will be merged with the `buildEnvVars` property, with the `buildEnvVars` having a precedence.\n\n*Note*: The project descriptor path should be relative to what is set in the [path](#path) property. If the `path` property is pointing to a zip archive (e.g. jar file), project descriptor path will be relative to the root of the workspace.\n\n*Note*: Inline buildpacks (see [specification](https://buildpacks.io/docs/reference/config/project-descriptor/#build-_table-optional_)) are not supported yet.\n")
 	cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", os.Getenv("PIPER_dockerConfigJSON"), "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).")
 	cmd.Flags().StringSliceVar(&stepConfig.CustomTLSCertificateLinks, "customTlsCertificateLinks", []string{}, "List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates.")
 	cmd.Flags().StringSliceVar(&stepConfig.AdditionalTags, "additionalTags", []string{}, "List of tags which will be pushed to the registry (additionally to the provided `containerImageTag`), e.g. \"latest\".")
 
+	cmd.Flags().StringSliceVar(&stepConfig.PreserveFiles, "preserveFiles", []string{}, "List of globs, for keeping build results in the Jenkins workspace.\n\n*Note*: globs will be calculated relative to the [path](#path) property.\n")
+
 	cmd.MarkFlagRequired("containerImageTag")
 	cmd.MarkFlagRequired("containerRegistryUrl")
 }
@@ -324,6 +327,15 @@ func cnbBuildMetadata() config.StepData {
 						Mandatory:   false,
 						Aliases:     []config.Alias{{Name: "images"}},
 					},
+					{
+						Name:        "preserveFiles",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "[]string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     []string{},
+					},
 				},
 			},
 			Containers: []config.Container{

integration/integration_cnb_test.go

@@ -233,3 +233,40 @@ func TestMultiImage(t *testing.T) {
 	container.assertHasOutput(t, "Saving localhost:5000/go-app:v1.0.0...")
 	container.terminate(t)
 }
+
+func TestPreserveFiles(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	registryContainer := setupDockerRegistry(t, ctx)
+	defer registryContainer.Terminate(ctx)
+
+	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
+		Image:   "paketobuildpacks/builder:full",
+		User:    "cnb",
+		TestDir: []string{"testdata", "TestCnbIntegration"},
+		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
+	})
+
+	container.whenRunningPiperCommand("cnbBuild", "--customConfig", "config_preserve_files.yml")
+	container.assertHasFile(t, "/project/project/node_modules/base/README.md")
+	container.assertHasFile(t, "/project/project/package-lock.json")
+	container.terminate(t)
+}
+
+func TestPreserveFilesIgnored(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	registryContainer := setupDockerRegistry(t, ctx)
+	defer registryContainer.Terminate(ctx)
+
+	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
+		Image:   "paketobuildpacks/builder:full",
+		User:    "cnb",
+		TestDir: []string{"testdata", "TestCnbIntegration"},
+		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
+	})
+
+	container.whenRunningPiperCommand("cnbBuild", "--customConfig", "config_preserve_files.yml", "--path", "zip/go.zip", "--containerImageName", "go-zip")
+	container.assertHasOutput(t, "skipping preserving files because the source")
+	container.terminate(t)
+}

integration/testdata/TestCnbIntegration/config_preserve_files.yml

@@ -0,0 +1,10 @@
+general:
+  verbose: true
+steps:
+  cnbBuild:
+    containerRegistryUrl: localhost:5000
+    containerImageTag: latest
+    path: project
+    preserveFiles:
+    - "node_modules/base"
+    - "package-lock.json"

pkg/cnbutils/copy_project.go

@@ -0,0 +1,93 @@
+package cnbutils
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/pkg/errors"
+	ignore "github.com/sabhiram/go-gitignore"
+)
+
+func CopyProject(source, target string, include, exclude *ignore.GitIgnore, utils BuildUtils) error {
+	sourceFiles, _ := utils.Glob(path.Join(source, "**"))
+	for _, sourceFile := range sourceFiles {
+		relPath, err := filepath.Rel(source, sourceFile)
+		if err != nil {
+			log.SetErrorCategory(log.ErrorBuild)
+			return errors.Wrapf(err, "Calculating relative path for '%s' failed", sourceFile)
+		}
+		if !isIgnored(relPath, include, exclude) {
+			target := path.Join(target, strings.ReplaceAll(sourceFile, source, ""))
+			dir, err := utils.DirExists(sourceFile)
+			if err != nil {
+				log.SetErrorCategory(log.ErrorBuild)
+				return errors.Wrapf(err, "Checking file info '%s' failed", target)
+			}
+
+			if dir {
+				err = utils.MkdirAll(target, os.ModePerm)
+				if err != nil {
+					log.SetErrorCategory(log.ErrorBuild)
+					return errors.Wrapf(err, "Creating directory '%s' failed", target)
+				}
+			} else {
+				log.Entry().Debugf("Copying '%s' to '%s'", sourceFile, target)
+				err = copyFile(sourceFile, target, utils)
+				if err != nil {
+					log.SetErrorCategory(log.ErrorBuild)
+					return errors.Wrapf(err, "Copying '%s' to '%s' failed", sourceFile, target)
+				}
+			}
+
+		}
+	}
+	return nil
+}
+
+func copyFile(source, target string, utils BuildUtils) error {
+	targetDir := filepath.Dir(target)
+
+	exists, err := utils.DirExists(targetDir)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		log.Entry().Debugf("Creating directory %s", targetDir)
+		err = utils.MkdirAll(targetDir, os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err = utils.Copy(source, target)
+	return err
+}
+
+func isIgnored(find string, include, exclude *ignore.GitIgnore) bool {
+	if exclude != nil {
+		filtered := exclude.MatchesPath(find)
+
+		if filtered {
+			log.Entry().Debugf("%s matches exclude pattern, ignoring", find)
+			return true
+		}
+	}
+
+	if include != nil {
+		filtered := !include.MatchesPath(find)
+
+		if filtered {
+			log.Entry().Debugf("%s doesn't match include pattern, ignoring", find)
+			return true
+		} else {
+			log.Entry().Debugf("%s matches include pattern", find)
+			return false
+		}
+	}
+
+	return false
+}

pkg/cnbutils/copy_project_test.go

@@ -0,0 +1,36 @@
+package cnbutils_test
+
+import (
+	"testing"
+
+	"github.com/SAP/jenkins-library/pkg/cnbutils"
+	"github.com/SAP/jenkins-library/pkg/mock"
+	ignore "github.com/sabhiram/go-gitignore"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCopyProject(t *testing.T) {
+	t.Run("copies file according to doublestart globs", func(t *testing.T) {
+		mockUtils := &cnbutils.MockUtils{
+			FilesMock: &mock.FilesMock{},
+		}
+		mockUtils.AddFile("workdir/src/test.yaml", []byte(""))
+		mockUtils.AddFile("workdir/src/subdir1/test2.yaml", []byte(""))
+		mockUtils.AddFile("workdir/src/subdir1/subdir2/test3.yaml", []byte(""))
+		err := cnbutils.CopyProject("workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**/*.yaml"}...), nil, mockUtils)
+		assert.NoError(t, err)
+		assert.True(t, mockUtils.HasCopiedFile("workdir/src/test.yaml", "/dest/test.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/test2.yaml", "/dest/subdir1/test2.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/subdir2/test3.yaml", "/dest/subdir1/subdir2/test3.yaml"))
+	})
+
+	t.Run("copies file according to simple globs", func(t *testing.T) {
+		mockUtils := &cnbutils.MockUtils{
+			FilesMock: &mock.FilesMock{},
+		}
+		mockUtils.AddFile("src/test.yaml", []byte(""))
+		err := cnbutils.CopyProject("src", "/dest", ignore.CompileIgnoreLines([]string{"*.yaml"}...), nil, mockUtils)
+		assert.NoError(t, err)
+		assert.True(t, mockUtils.HasCopiedFile("src/test.yaml", "/dest/test.yaml"))
+	})
+}

resources/metadata/cnbBuild.yaml

@@ -118,8 +118,9 @@ spec:
           Parameters passed to the cnbBuild step will take precedence over the parameters set in the project.toml file, except the `env` block.
           Environment variables declared in a project descriptor file, will be merged with the `buildEnvVars` property, with the `buildEnvVars` having a precedence.
 
-          Note: The project descriptor path should be relative to what is set in the [path](#path) property. If the `path` property is pointing to a zip archive (e.g. jar file), project descriptor path will be relative to the root of the workspace.
-          Note: Inline buildpacks (see [specification](https://buildpacks.io/docs/reference/config/project-descriptor/#build-_table-optional_)) are not supported yet.
+          *Note*: The project descriptor path should be relative to what is set in the [path](#path) property. If the `path` property is pointing to a zip archive (e.g. jar file), project descriptor path will be relative to the root of the workspace.
+
+          *Note*: Inline buildpacks (see [specification](https://buildpacks.io/docs/reference/config/project-descriptor/#build-_table-optional_)) are not supported yet.
         default: project.toml
         scope:
           - PARAMETERS
@@ -219,6 +220,16 @@ spec:
           - PARAMETERS
           - STAGES
           - STEPS
+      - name: preserveFiles
+        type: "[]string"
+        description: |
+          List of globs, for keeping build results in the Jenkins workspace.
+
+          *Note*: globs will be calculated relative to the [path](#path) property.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
   outputs:
     resources:
       - name: commonPipelineEnvironment