feat(kanikoExecute): Trim names for multi-image builds (#3617)

* feat(kanikoExecute): Trim names for multi-image builds * chore: fix yaml linting issue
2025-03-03 15:02:35 +02:00 · 2022-03-11 09:47:44 +01:00 · 2022-03-11 09:47:44 +01:00 · 2a56723d40
commit 2a56723d40
parent 809a90ee27
5 changed files with 174 additions and 47 deletions
--- a/cmd/kanikoExecute.go
+++ b/cmd/kanikoExecute.go
@ -122,7 +122,7 @@ func runKanikoExecute(config *kanikoExecuteOptions, telemetryData *telemetry.Cus

 			if config.ContainerMultiImageBuild {
 				log.Entry().Debugf("Multi-image build activated for image name '%v'", config.ContainerImageName)
-				imageListWithFilePath, err := docker.ImageListWithFilePath(config.ContainerImageName, config.ContainerMultiImageBuildExcludes, fileUtils)
+				imageListWithFilePath, err := docker.ImageListWithFilePath(config.ContainerImageName, config.ContainerMultiImageBuildExcludes, config.ContainerMultiImageBuildTrimDir, fileUtils)
 				if err != nil {
 					return fmt.Errorf("failed to identify image list for multi image build: %w", err)
 				}
--- a/cmd/kanikoExecute_generated.go
+++ b/cmd/kanikoExecute_generated.go
@ -20,12 +20,13 @@ import (
 type kanikoExecuteOptions struct {
 	BuildOptions                     []string `json:"buildOptions,omitempty"`
 	BuildSettingsInfo                string   `json:"buildSettingsInfo,omitempty"`
-	ContainerMultiImageBuild         bool     `json:"containerMultiImageBuild,omitempty"`
-	ContainerMultiImageBuildExcludes []string `json:"containerMultiImageBuildExcludes,omitempty"`
 	ContainerBuildOptions            string   `json:"containerBuildOptions,omitempty"`
 	ContainerImage                   string   `json:"containerImage,omitempty"`
 	ContainerImageName               string   `json:"containerImageName,omitempty"`
 	ContainerImageTag                string   `json:"containerImageTag,omitempty"`
+	ContainerMultiImageBuild         bool     `json:"containerMultiImageBuild,omitempty"`
+	ContainerMultiImageBuildExcludes []string `json:"containerMultiImageBuildExcludes,omitempty"`
+	ContainerMultiImageBuildTrimDir  string   `json:"containerMultiImageBuildTrimDir,omitempty"`
 	ContainerPreparationCommand      string   `json:"containerPreparationCommand,omitempty"`
 	ContainerRegistryURL             string   `json:"containerRegistryUrl,omitempty"`
 	CustomTLSCertificateLinks        []string `json:"customTlsCertificateLinks,omitempty"`
@ -103,10 +104,59 @@ All images will get the same "root" name and the same versioning.<br />
 **Thus, this is not suitable to be used for a monorepo approach!** For monorepos you need to use a build tool natively capable to take care for monorepos
 or implement a custom logic and for example execute this ` + "`" + `kanikoExecute` + "`" + ` step multiple times in your custom pipeline.

-You can activate multiple builds using the parameters
+You can activate multiple builds using the parameter [containerMultiImageBuild](#containermultiimagebuild)

-* [containerMultiImageBuild](#containermultiimagebuild) for activation
-* [containerMultiImageBuildExcludes](#containermultiimagebuildexcludes) for defining excludes`,
+Behavior can be adapted using:
+
+* [containerMultiImageBuildExcludes](#containermultiimagebuildexcludes) for defining excludes
+* [containerMultiImageBuildTrimDir](#containermultiimagebuildtrimdir) for removing parent directory part from image name
+
+Examples:
+
+#### Multiple containers in sub directories
+
+Configuration as follows:
+
+` + "`" + `` + "`" + `` + "`" + `
+general:
+  containerImageName: myImage
+steps:
+  kanikoExecute:
+    containerMultiImageBuild: true
+` + "`" + `` + "`" + `` + "`" + `
+
+Following Dockerfiles are available in the repository:
+
+* sub1/Dockerfile
+* sub2/Dockerfile
+
+Following final image names will be built:
+
+* ` + "`" + `myImage-sub1` + "`" + `
+* ` + "`" + `myImage-sub2` + "`" + `
+
+#### Multiple containers in sub directories while trimming a directory part
+
+Configuration as follows:
+
+` + "`" + `` + "`" + `` + "`" + `
+general:
+  containerImageName: myImage
+steps:
+  kanikoExecute:
+    containerMultiImageBuild: true
+    containerMultiImageBuildTrimDir: .ci
+` + "`" + `` + "`" + `` + "`" + `
+
+Following Dockerfiles are available in the repository:
+
+* .ci/sub1/Dockerfile
+* .ci/sub2/Dockerfile
+
+Following final image names will be built:
+
+* ` + "`" + `myImage-sub1` + "`" + `
+* ` + "`" + `myImage-sub2` + "`" + ``,
 		PreRunE: func(cmd *cobra.Command, _ []string) error {
 			startTime = time.Now()
 			log.SetStepName(STEP_NAME)
@ -185,12 +235,13 @@ You can activate multiple builds using the parameters
 func addKanikoExecuteFlags(cmd *cobra.Command, stepConfig *kanikoExecuteOptions) {
 	cmd.Flags().StringSliceVar(&stepConfig.BuildOptions, "buildOptions", []string{`--skip-tls-verify-pull`, `--ignore-path=/workspace`, `--ignore-path=/busybox`}, "Defines a list of build options for the [kaniko](https://github.com/GoogleContainerTools/kaniko) build.")
 	cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "Build settings info is typically filled by the step automatically to create information about the build settings that were used during the mta build. This information is typically used for compliance related processes.")
-	cmd.Flags().BoolVar(&stepConfig.ContainerMultiImageBuild, "containerMultiImageBuild", false, "Defines if multiple containers should be build. Dockerfiles are used using the pattern **/Dockerfile*. Excludes can be defined via [`containerMultiImageBuildExcludes`](#containermultiimagebuildexscludes).")
-	cmd.Flags().StringSliceVar(&stepConfig.ContainerMultiImageBuildExcludes, "containerMultiImageBuildExcludes", []string{}, "Defines a list of Dockerfile paths to exclude from the build when using [`containerMultiImageBuild`](#containermultiimagebuild).")
 	cmd.Flags().StringVar(&stepConfig.ContainerBuildOptions, "containerBuildOptions", os.Getenv("PIPER_containerBuildOptions"), "Deprected, please use buildOptions. Defines the build options for the [kaniko](https://github.com/GoogleContainerTools/kaniko) build.")
 	cmd.Flags().StringVar(&stepConfig.ContainerImage, "containerImage", os.Getenv("PIPER_containerImage"), "Defines the full name of the Docker image to be created including registry, image name and tag like `my.docker.registry/path/myImageName:myTag`. If left empty, image will not be pushed.")
 	cmd.Flags().StringVar(&stepConfig.ContainerImageName, "containerImageName", os.Getenv("PIPER_containerImageName"), "Name of the container which will be built - will be used instead of parameter `containerImage`")
 	cmd.Flags().StringVar(&stepConfig.ContainerImageTag, "containerImageTag", os.Getenv("PIPER_containerImageTag"), "Tag of the container which will be built - will be used instead of parameter `containerImage`")
+	cmd.Flags().BoolVar(&stepConfig.ContainerMultiImageBuild, "containerMultiImageBuild", false, "Defines if multiple containers should be build. Dockerfiles are used using the pattern **/Dockerfile*. Excludes can be defined via [`containerMultiImageBuildExcludes`](#containermultiimagebuildexscludes).")
+	cmd.Flags().StringSliceVar(&stepConfig.ContainerMultiImageBuildExcludes, "containerMultiImageBuildExcludes", []string{}, "Defines a list of Dockerfile paths to exclude from the build when using [`containerMultiImageBuild`](#containermultiimagebuild).")
+	cmd.Flags().StringVar(&stepConfig.ContainerMultiImageBuildTrimDir, "containerMultiImageBuildTrimDir", os.Getenv("PIPER_containerMultiImageBuildTrimDir"), "Defines a trailing directory part which should not be considered in the final image name.")
 	cmd.Flags().StringVar(&stepConfig.ContainerPreparationCommand, "containerPreparationCommand", `rm -f /kaniko/.docker/config.json`, "Defines the command to prepare the Kaniko container. By default the contained credentials are removed in order to allow anonymous access to container registries.")
 	cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "http(s) url of the Container registry where the image should be pushed to - will be used instead of parameter `containerImage`")
 	cmd.Flags().StringSliceVar(&stepConfig.CustomTLSCertificateLinks, "customTlsCertificateLinks", []string{}, "List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates.")
@ -238,24 +289,6 @@ func kanikoExecuteMetadata() config.StepData {
 						Aliases:   []config.Alias{},
 						Default:   os.Getenv("PIPER_buildSettingsInfo"),
 					},
-					{
-						Name:        "containerMultiImageBuild",
-						ResourceRef: []config.ResourceReference{},
-						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
-						Type:        "bool",
-						Mandatory:   false,
-						Aliases:     []config.Alias{},
-						Default:     false,
-					},
-					{
-						Name:        "containerMultiImageBuildExcludes",
-						ResourceRef: []config.ResourceReference{},
-						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
-						Type:        "[]string",
-						Mandatory:   false,
-						Aliases:     []config.Alias{},
-						Default:     []string{},
-					},
 					{
 						Name:        "containerBuildOptions",
 						ResourceRef: []config.ResourceReference{},
@ -297,6 +330,33 @@ func kanikoExecuteMetadata() config.StepData {
 						Aliases:   []config.Alias{{Name: "artifactVersion"}},
 						Default:   os.Getenv("PIPER_containerImageTag"),
 					},
+					{
+						Name:        "containerMultiImageBuild",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:        "bool",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     false,
+					},
+					{
+						Name:        "containerMultiImageBuildExcludes",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:        "[]string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     []string{},
+					},
+					{
+						Name:        "containerMultiImageBuildTrimDir",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_containerMultiImageBuildTrimDir"),
+					},
 					{
 						Name:        "containerPreparationCommand",
 						ResourceRef: []config.ResourceReference{},
--- a/pkg/docker/docker.go
+++ b/pkg/docker/docker.go
@ -174,7 +174,7 @@ func (c *Client) TarImage(writer io.Writer, image pkgutil.Image) error {
 // * Dockerfile: `imageName`
 // * sub1/Dockerfile: `imageName-sub1`
 // * sub2/Dockerfile_proxy: `imageName-sub2-proxy`
-func ImageListWithFilePath(imageName string, excludes []string, utils piperutils.FileUtils) (map[string]string, error) {
+func ImageListWithFilePath(imageName string, excludes []string, trimDir string, utils piperutils.FileUtils) (map[string]string, error) {

 	imageList := map[string]string{}

@ -200,7 +200,16 @@ func ImageListWithFilePath(imageName string, excludes []string, utils piperutils
 		} else {
 			var finalName string
 			if base := filepath.Base(dockerfilePath); base == "Dockerfile" {
-				finalName = fmt.Sprintf("%v-%v", imageName, strings.ReplaceAll(filepath.Dir(dockerfilePath), string(filepath.Separator), "-"))
+				subName := strings.ReplaceAll(filepath.Dir(dockerfilePath), string(filepath.Separator), "-")
+				if len(trimDir) > 0 {
+					// allow to remove trailing sub directories
+					// example .ci/app/Dockerfile
+					// with trimDir = .ci/ imagename would only contain app part.
+					subName = strings.TrimPrefix(subName, strings.ReplaceAll(trimDir, "/", "-"))
+					// make sure that subName does not start with a - (e.g. due not configuring trailing slash for trimDir)
+					subName = strings.TrimPrefix(subName, "-")
+				}
+				finalName = fmt.Sprintf("%v-%v", imageName, subName)
 			} else {
 				parts := strings.FieldsFunc(base, func(separator rune) bool {
 					return separator == []rune("-")[0] || separator == []rune("_")[0]
--- a/pkg/docker/docker_test.go
+++ b/pkg/docker/docker_test.go
@ -140,12 +140,14 @@ func TestImageListWithFilePath(t *testing.T) {
 	tt := []struct {
 		name          string
 		excludes      []string
+		trimDir       string
 		fileList      []string
 		expected      map[string]string
 		expectedError error
 	}{
 		{name: "Dockerfile only", fileList: []string{"Dockerfile"}, expected: map[string]string{imageName: "Dockerfile"}},
 		{name: "Dockerfile in subdir", fileList: []string{"sub/Dockerfile"}, expected: map[string]string{fmt.Sprintf("%v-sub", imageName): filepath.FromSlash("sub/Dockerfile")}},
+		{name: "Dockerfile in subdir excluding top dir", fileList: []string{".ci/sub/Dockerfile"}, trimDir: ".ci", expected: map[string]string{fmt.Sprintf("%v-sub", imageName): filepath.FromSlash(".ci/sub/Dockerfile")}},
 		{name: "Dockerfiles in multiple subdirs & parent", fileList: []string{"Dockerfile", "sub1/Dockerfile", "sub2/Dockerfile"}, expected: map[string]string{fmt.Sprintf("%v", imageName): filepath.FromSlash("Dockerfile"), fmt.Sprintf("%v-sub1", imageName): filepath.FromSlash("sub1/Dockerfile"), fmt.Sprintf("%v-sub2", imageName): filepath.FromSlash("sub2/Dockerfile")}},
 		{name: "Dockerfiles in multiple subdirs & parent - with excludes", excludes: []string{"Dockerfile"}, fileList: []string{"Dockerfile", "sub1/Dockerfile", "sub2/Dockerfile"}, expected: map[string]string{fmt.Sprintf("%v-sub1", imageName): filepath.FromSlash("sub1/Dockerfile"), fmt.Sprintf("%v-sub2", imageName): filepath.FromSlash("sub2/Dockerfile")}},
 		{name: "Dockerfiles with extensions", fileList: []string{"Dockerfile_main", "Dockerfile_sub1", "Dockerfile_sub2"}, expected: map[string]string{fmt.Sprintf("%v-main", imageName): filepath.FromSlash("Dockerfile_main"), fmt.Sprintf("%v-sub1", imageName): filepath.FromSlash("Dockerfile_sub1"), fmt.Sprintf("%v-sub2", imageName): filepath.FromSlash("Dockerfile_sub2")}},
@ -161,7 +163,7 @@ func TestImageListWithFilePath(t *testing.T) {
 				fileMock.AddFile(file, []byte("someContent"))
 			}

-			imageList, err := ImageListWithFilePath(imageName, test.excludes, &fileMock)
+			imageList, err := ImageListWithFilePath(imageName, test.excludes, test.trimDir, &fileMock)

 			if test.expectedError != nil {
 				assert.EqualError(t, err, fmt.Sprint(test.expectedError))
--- a/resources/metadata/kanikoExecute.yaml
+++ b/resources/metadata/kanikoExecute.yaml
@ -13,10 +13,59 @@ metadata:
    **Thus, this is not suitable to be used for a monorepo approach!** For monorepos you need to use a build tool natively capable to take care for monorepos
    or implement a custom logic and for example execute this `kanikoExecute` step multiple times in your custom pipeline.

-    You can activate multiple builds using the parameters
+    You can activate multiple builds using the parameter [containerMultiImageBuild](#containermultiimagebuild)
+
+    Behavior can be adapted using:

-    * [containerMultiImageBuild](#containermultiimagebuild) for activation
    * [containerMultiImageBuildExcludes](#containermultiimagebuildexcludes) for defining excludes
+    * [containerMultiImageBuildTrimDir](#containermultiimagebuildtrimdir) for removing parent directory part from image name
+
+    Examples:
+
+    #### Multiple containers in sub directories
+
+    Configuration as follows:
+
+    ```
+    general:
+      containerImageName: myImage
+    steps:
+      kanikoExecute:
+        containerMultiImageBuild: true
+    ```
+
+    Following Dockerfiles are available in the repository:
+
+    * sub1/Dockerfile
+    * sub2/Dockerfile
+
+    Following final image names will be built:
+
+    * `myImage-sub1`
+    * `myImage-sub2`
+
+    #### Multiple containers in sub directories while trimming a directory part
+
+    Configuration as follows:
+
+    ```
+    general:
+      containerImageName: myImage
+    steps:
+      kanikoExecute:
+        containerMultiImageBuild: true
+        containerMultiImageBuildTrimDir: .ci
+    ```
+
+    Following Dockerfiles are available in the repository:
+
+    * .ci/sub1/Dockerfile
+    * .ci/sub2/Dockerfile
+
+    Following final image names will be built:
+
+    * `myImage-sub1`
+    * `myImage-sub2`

 spec:
  inputs:
@ -48,22 +97,6 @@ spec:
        resourceRef:
          - name: commonPipelineEnvironment
            param: custom/buildSettingsInfo
-      - name: containerMultiImageBuild
-        type: bool
-        description: Defines if multiple containers should be build. Dockerfiles are used using the pattern **/Dockerfile*. Excludes can be defined via [`containerMultiImageBuildExcludes`](#containermultiimagebuildexscludes).
-        scope:
-          - GENERAL
-          - PARAMETERS
-          - STAGES
-          - STEPS
-      - name: containerMultiImageBuildExcludes
-        type: '[]string'
-        description: Defines a list of Dockerfile paths to exclude from the build when using [`containerMultiImageBuild`](#containermultiimagebuild).
-        scope:
-          - GENERAL
-          - PARAMETERS
-          - STAGES
-          - STEPS
      - name: containerBuildOptions
        type: string
        description: Deprected, please use buildOptions. Defines the build options for the [kaniko](https://github.com/GoogleContainerTools/kaniko) build.
@ -104,6 +137,29 @@ spec:
        resourceRef:
          - name: commonPipelineEnvironment
            param: artifactVersion
+      - name: containerMultiImageBuild
+        type: bool
+        description: Defines if multiple containers should be build. Dockerfiles are used using the pattern **/Dockerfile*. Excludes can be defined via [`containerMultiImageBuildExcludes`](#containermultiimagebuildexscludes).
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: containerMultiImageBuildExcludes
+        type: '[]string'
+        description: Defines a list of Dockerfile paths to exclude from the build when using [`containerMultiImageBuild`](#containermultiimagebuild).
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: containerMultiImageBuildTrimDir
+        type: 'string'
+        description: Defines a trailing directory part which should not be considered in the final image name.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
      - name: containerPreparationCommand
        type: string
        description: Defines the command to prepare the Kaniko container. By default the contained credentials are removed in order to allow anonymous access to container registries.