From d8dacda12120f644f231dc46084678db371974e3 Mon Sep 17 00:00:00 2001 From: Pavel Busko Date: Fri, 30 Jun 2023 14:02:35 +0200 Subject: [PATCH] feat(cnbBuild): support setting registry username and password via parameters (#4426) * feat(cnbBuild): support setting registry username and password via parameters * fix gitops integration test assertion Co-authored-by: Pavel Busko * Update integration/integration_gitops_test.go --------- Co-authored-by: Ralf Pannemans --- cmd/cnbBuild.go | 18 ++++++++- cmd/cnbBuild_generated.go | 37 ++++++++++++++++--- integration/docker_test_executor.go | 4 +- integration/integration_cnb_test.go | 28 +++++++------- integration/integration_gitops_test.go | 3 +- .../testdata/TestCnbIntegration/config.json | 5 +++ resources/metadata/cnbBuild.yaml | 28 +++++++++++++- 7 files changed, 97 insertions(+), 26 deletions(-) create mode 100644 integration/testdata/TestCnbIntegration/config.json diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go index 74bb30301..4b49b51a7 100644 --- a/cmd/cnbBuild.go +++ b/cmd/cnbBuild.go @@ -385,10 +385,26 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, err = renameDockerConfig(config, utils) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) - return errors.Wrapf(err, "failed to rename DockerConfigJSON file '%v'", config.DockerConfigJSON) + return errors.Wrapf(err, "failed to rename DockerConfigJSON file '%s'", config.DockerConfigJSON) } } + if config.ContainerRegistryUser != "" && config.ContainerRegistryPassword != "" { + log.Entry().Debug("enhancing docker config with the provided credentials") + if config.DockerConfigJSON == "" { + config.DockerConfigJSON = "/tmp/config.json" + } + log.Entry().Debugf("using docker config file %q", config.DockerConfigJSON) + + _, err = docker.CreateDockerConfigJSON(config.ContainerRegistryURL, config.ContainerRegistryUser, config.ContainerRegistryPassword, "", config.DockerConfigJSON, utils) + if err != nil { + log.SetErrorCategory(log.ErrorBuild) + return errors.Wrapf(err, "failed to update DockerConfigJSON file %q", config.DockerConfigJSON) + } + + log.Entry().Debugf("docker config %q has been updated", config.DockerConfigJSON) + } + mergedConfigs, err := processConfigs(*config, config.MultipleImages) if err != nil { return errors.Wrap(err, "failed to process config") diff --git a/cmd/cnbBuild_generated.go b/cmd/cnbBuild_generated.go index 45c8f7ec0..c714123ed 100644 --- a/cmd/cnbBuild_generated.go +++ b/cmd/cnbBuild_generated.go @@ -26,6 +26,8 @@ type cnbBuildOptions struct { ContainerImageAlias string `json:"containerImageAlias,omitempty"` ContainerImageTag string `json:"containerImageTag,omitempty"` ContainerRegistryURL string `json:"containerRegistryUrl,omitempty"` + ContainerRegistryUser string `json:"containerRegistryUser,omitempty"` + ContainerRegistryPassword string `json:"containerRegistryPassword,omitempty"` Buildpacks []string `json:"buildpacks,omitempty"` BuildEnvVars map[string]interface{} `json:"buildEnvVars,omitempty"` Path string `json:"path,omitempty"` @@ -224,6 +226,8 @@ func addCnbBuildFlags(cmd *cobra.Command, stepConfig *cnbBuildOptions) { cmd.Flags().StringVar(&stepConfig.ContainerImageAlias, "containerImageAlias", os.Getenv("PIPER_containerImageAlias"), "Logical name used for this image.\n") cmd.Flags().StringVar(&stepConfig.ContainerImageTag, "containerImageTag", os.Getenv("PIPER_containerImageTag"), "Tag of the container which will be built") cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "Container registry where the image should be pushed to.\n\n**Note**: `containerRegistryUrl` should include only the domain. If you want to publish an image under `docker.io/example/my-image`, you must set `containerRegistryUrl: \"docker.io\"` and `containerImageName: \"example/my-image\"`.\n") + cmd.Flags().StringVar(&stepConfig.ContainerRegistryUser, "containerRegistryUser", os.Getenv("PIPER_containerRegistryUser"), "Username of the container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON`, then the existing file will be enhanced") + cmd.Flags().StringVar(&stepConfig.ContainerRegistryPassword, "containerRegistryPassword", os.Getenv("PIPER_containerRegistryPassword"), "Password of the container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON`, then the existing file will be enhanced") cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`.") cmd.Flags().StringVar(&stepConfig.Path, "path", os.Getenv("PIPER_path"), "Glob that should either point to a directory with your sources or one artifact in zip format.\nThis property determines the input to the buildpack.\n") @@ -308,6 +312,34 @@ func cnbBuildMetadata() config.StepData { Aliases: []config.Alias{{Name: "dockerRegistryUrl"}}, Default: os.Getenv("PIPER_containerRegistryUrl"), }, + { + Name: "containerRegistryUser", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/repositoryUsername", + }, + }, + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{{Name: "dockerRegistryUser"}}, + Default: os.Getenv("PIPER_containerRegistryUser"), + }, + { + Name: "containerRegistryPassword", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/repositoryPassword", + }, + }, + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{{Name: "dockerRegistryPassword"}}, + Default: os.Getenv("PIPER_containerRegistryPassword"), + }, { Name: "buildpacks", ResourceRef: []config.ResourceReference{ @@ -351,11 +383,6 @@ func cnbBuildMetadata() config.StepData { { Name: "dockerConfigJSON", ResourceRef: []config.ResourceReference{ - { - Name: "commonPipelineEnvironment", - Param: "custom/dockerConfigJSON", - }, - { Name: "dockerConfigJsonCredentialsId", Type: "secret", diff --git a/integration/docker_test_executor.go b/integration/docker_test_executor.go index 87f944434..890610410 100644 --- a/integration/docker_test_executor.go +++ b/integration/docker_test_executor.go @@ -293,9 +293,7 @@ func (d *IntegrationTestDockerExecRunner) assertFileContentEquals(t *testing.T, t.Fatalf("unable to get tar file content: %s", err) } - if !strings.Contains(str.String(), contentWant) { - assert.Equal(t, str.String(), contentWant, fmt.Sprintf("Unexpected content of file '%s'", fileWant)) - } + assert.Equal(t, str.String(), contentWant, fmt.Sprintf("Unexpected content of file '%s'", fileWant)) } func (d *IntegrationTestDockerExecRunner) terminate(t *testing.T) { diff --git a/integration/integration_cnb_test.go b/integration/integration_cnb_test.go index 39be5ee04..edfe98deb 100644 --- a/integration/integration_cnb_test.go +++ b/integration/integration_cnb_test.go @@ -50,7 +50,6 @@ func TestCNBIntegrationNPMProject(t *testing.T) { "PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content", }, }) - defer container.terminate(t) container2 := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: baseBuilder, @@ -61,9 +60,8 @@ func TestCNBIntegrationNPMProject(t *testing.T) { "PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content", }, }) - defer container2.terminate(t) - err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--path", "TestCnbIntegration/project", "--customConfig", "TestCnbIntegration/config.yml", "--containerImageName", "node", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--defaultProcess", "greeter") + err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--path", "TestCnbIntegration/project", "--customConfig", "TestCnbIntegration/config.yml", "--containerImageName", "node", "--containerImageTag", "0.0.1", "--dockerConfigJSON", "TestCnbIntegration/config.json", "--containerRegistryUrl", registryURL, "--containerRegistryUser", "foo", "--containerRegistryPassword", "bar", "--defaultProcess", "greeter") assert.NoError(t, err) container.assertHasOutput(t, "running command: /cnb/lifecycle/creator") container.assertHasOutput(t, "Selected Node Engine version (using BP_NODE_VERSION): 16") @@ -72,9 +70,10 @@ func TestCNBIntegrationNPMProject(t *testing.T) { container.assertHasOutput(t, "Setting default process type 'greeter'") container.assertHasOutput(t, "*** Images (sha256:") container.assertHasOutput(t, "SUCCESS") + container.assertFileContentEquals(t, "/project/TestCnbIntegration/config.json", "{\"auths\":{\"localhost:5000\":{\"auth\":\"Zm9vOmJhcg==\"},\"test.registry.io\":{}}}") container.terminate(t) - err = container2.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--path", "TestCnbIntegration/project", "--customConfig", "TestCnbIntegration/config.yml", "--containerImageName", "node", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--projectDescriptor", "project-with-id.toml") + err = container2.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--path", "TestCnbIntegration/project", "--customConfig", "TestCnbIntegration/config.yml", "--containerImageName", "node", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--containerRegistryUser", "foo", "--containerRegistryPassword", "bar", "--projectDescriptor", "project-with-id.toml") assert.NoError(t, err) container2.assertHasOutput(t, "running command: /cnb/lifecycle/creator") container2.assertHasOutput(t, "Selected Node Engine version (using BP_NODE_VERSION): 16") @@ -82,6 +81,7 @@ func TestCNBIntegrationNPMProject(t *testing.T) { container2.assertHasOutput(t, fmt.Sprintf("Saving %s/node:0.0.1", registryURL)) container2.assertHasOutput(t, "*** Images (sha256:") container2.assertHasOutput(t, "SUCCESS") + container2.assertFileContentEquals(t, "/tmp/config.json", "{\"auths\":{\"localhost:5000\":{\"auth\":\"Zm9vOmJhcg==\"}}}") container2.terminate(t) } @@ -97,7 +97,6 @@ func TestCNBIntegrationProjectDescriptor(t *testing.T) { TestDir: []string{"testdata", "TestCnbIntegration", "project"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL) assert.NoError(t, err) @@ -114,6 +113,7 @@ func TestCNBIntegrationProjectDescriptor(t *testing.T) { "*** Images (sha256:", "SUCCESS", ) + container.terminate(t) } func TestCNBIntegrationZipPath(t *testing.T) { @@ -128,7 +128,6 @@ func TestCNBIntegrationZipPath(t *testing.T) { TestDir: []string{"testdata", "TestCnbIntegration", "zip"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--path", "go.zip", "--createBOM") assert.NoError(t, err) @@ -143,6 +142,7 @@ func TestCNBIntegrationZipPath(t *testing.T) { "syft packages registry:localhost:5000/not-found:0.0.1 -o cyclonedx-xml --file bom-docker-0.xml -q", ) container.assertHasFiles(t, "/project/bom-docker-0.xml") + container.terminate(t) } func TestCNBIntegrationNonZipPath(t *testing.T) { @@ -157,12 +157,12 @@ func TestCNBIntegrationNonZipPath(t *testing.T) { TestDir: []string{"testdata", "TestMtaIntegration", "npm"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--path", "mta.yaml") assert.Error(t, err) container.assertHasOutput(t, "Copying '/project/mta.yaml' into '/workspace' failed: application path must be a directory or zip") + container.terminate(t) } func TestCNBIntegrationNPMCustomBuildpacksFullProject(t *testing.T) { @@ -177,7 +177,6 @@ func TestCNBIntegrationNPMCustomBuildpacksFullProject(t *testing.T) { TestDir: []string{"testdata", "TestMtaIntegration", "npm"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--buildpacks", "gcr.io/paketo-buildpacks/nodejs:0.19.0", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL) assert.NoError(t, err) @@ -191,6 +190,7 @@ func TestCNBIntegrationNPMCustomBuildpacksFullProject(t *testing.T) { "*** Images (sha256:", "SUCCESS", ) + container.terminate(t) } func TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject(t *testing.T) { @@ -205,7 +205,6 @@ func TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject(t *testing.T) { TestDir: []string{"testdata", "TestMtaIntegration", "npm"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--buildpacks", "gcr.io/paketo-buildpacks/nodejs:0.19.0", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL) assert.NoError(t, err) @@ -218,6 +217,7 @@ func TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject(t *testing.T) { "*** Images (sha256:", "SUCCESS", ) + container.terminate(t) } func TestCNBIntegrationWrongBuilderProject(t *testing.T) { @@ -226,12 +226,12 @@ func TestCNBIntegrationWrongBuilderProject(t *testing.T) { Image: "nginx:latest", TestDir: []string{"testdata", "TestMtaIntegration", "npm"}, }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", "test") assert.Error(t, err) container.assertHasOutput(t, "the provided dockerImage is not a valid builder") + container.terminate(t) } func TestCNBIntegrationBindings(t *testing.T) { @@ -249,7 +249,6 @@ func TestCNBIntegrationBindings(t *testing.T) { "PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content", }, }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--customConfig", "TestCnbIntegration/config.yml", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--path", "TestMtaIntegration/maven") assert.Error(t, err) @@ -261,6 +260,7 @@ func TestCNBIntegrationBindings(t *testing.T) { ) container.assertFileContentEquals(t, "/tmp/platform/bindings/maven-settings/settings.xml", "invalid xml") container.assertFileContentEquals(t, "/tmp/platform/bindings/dynatrace/api-key", "api-key-content") + container.terminate(t) } func TestCNBIntegrationMultiImage(t *testing.T) { @@ -275,7 +275,6 @@ func TestCNBIntegrationMultiImage(t *testing.T) { TestDir: []string{"testdata", "TestCnbIntegration"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--customConfig", "config_multi_image.yml", "--createBOM") assert.NoError(t, err) @@ -295,6 +294,7 @@ func TestCNBIntegrationMultiImage(t *testing.T) { container.assertHasFiles(t, "/project/bom-docker-0.xml") container.assertHasFiles(t, "/project/bom-docker-1.xml") container.assertHasFiles(t, "/project/bom-docker-2.xml") + container.terminate(t) } func TestCNBIntegrationPreserveFiles(t *testing.T) { @@ -309,12 +309,12 @@ func TestCNBIntegrationPreserveFiles(t *testing.T) { TestDir: []string{"testdata", "TestCnbIntegration"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--customConfig", "config_preserve_files.yml") assert.NoError(t, err) container.assertHasFiles(t, "/project/project/node_modules/base/README.md", "/project/project/package-lock.json") + container.terminate(t) } func TestCNBIntegrationPreserveFilesIgnored(t *testing.T) { @@ -329,9 +329,9 @@ func TestCNBIntegrationPreserveFilesIgnored(t *testing.T) { TestDir: []string{"testdata", "TestCnbIntegration"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), }) - defer container.terminate(t) err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--customConfig", "config_preserve_files.yml", "--path", "zip/go.zip", "--containerImageName", "go-zip") assert.NoError(t, err) container.assertHasOutput(t, "skipping preserving files because the source") + container.terminate(t) } diff --git a/integration/integration_gitops_test.go b/integration/integration_gitops_test.go index 09aedb933..b29a32831 100644 --- a/integration/integration_gitops_test.go +++ b/integration/integration_gitops_test.go @@ -34,5 +34,6 @@ func TestGitOpsIntegrationUpdateDeployment(t *testing.T) { newName: image newTag: "456" apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization`) +kind: Kustomization +`) } diff --git a/integration/testdata/TestCnbIntegration/config.json b/integration/testdata/TestCnbIntegration/config.json new file mode 100644 index 000000000..e1cc1b7a1 --- /dev/null +++ b/integration/testdata/TestCnbIntegration/config.json @@ -0,0 +1,5 @@ +{ + "auths": { + "test.registry.io": {} + } +} \ No newline at end of file diff --git a/resources/metadata/cnbBuild.yaml b/resources/metadata/cnbBuild.yaml index 21f86af7c..809ca5dbd 100644 --- a/resources/metadata/cnbBuild.yaml +++ b/resources/metadata/cnbBuild.yaml @@ -95,6 +95,32 @@ spec: resourceRef: - name: commonPipelineEnvironment param: container/registryUrl + - name: containerRegistryUser + aliases: + - name: dockerRegistryUser + type: string + description: Username of the container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON`, then the existing file will be enhanced + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: container/repositoryUsername + - name: containerRegistryPassword + aliases: + - name: dockerRegistryPassword + type: string + description: Password of the container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON`, then the existing file will be enhanced + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: container/repositoryPassword - name: buildpacks type: "[]string" description: List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`. @@ -150,8 +176,6 @@ spec: - PARAMETERS secret: true resourceRef: - - name: commonPipelineEnvironment - param: custom/dockerConfigJSON - name: dockerConfigJsonCredentialsId type: secret - type: vaultSecretFile