1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-02-09 13:47:31 +02:00

feat: Add imagePushToRegistry step (#4609)

* imagePushToRegistry new step

* adding copy and push functionality

* including only copy correctly

* groovy step for imagePushToRegistry

* create .docker folder

* imagePushToRegistry new step

* adding copy and push functionality

* including only copy correctly

* groovy step for imagePushToRegistry

* create .docker folder

* fix CopyImage

* test

* test

* Correct docker config path

* Update

* Update

* Update

* Update

* Update

* Use creds from Vault

* Use creds from Vault

* Use creds from Vault

* Use creds from Vault

* Test

* Comment some logic

* Test: move regexp logic

* Test

* Update

* Update

* Clean up

* Update

* Update

* Update interface

* Rename function

* imagePushToRegistry: small refactoring (#4688)

* imagePushToRegistry new step

* adding copy and push functionality

* including only copy correctly

* groovy step for imagePushToRegistry

* create .docker folder

* Correct docker config path

* Update

* Update

* Update

* Update

* Update

* Use creds from Vault

* Use creds from Vault

* Use creds from Vault

* Use creds from Vault

* Test

* Comment some logic

* Test: move regexp logic

* Test

* Update

* Update

* Clean up

* Update

* Update

---------

Co-authored-by: Keshav <anil.keshav@sap.com>
Co-authored-by: Muhammadali Nazarov <muhammadalinazarov@gmail.com>

* Update step yaml file

* Update interface

* Rename func

* Update tests

* Update interface, create mock methods, update tests

* Update mock

* Add md file

* Fix groovy doc, unit test, go unit test

* Update

* Add unit tests

* Support tagLatest param

* Fetch source creds from Vault

* Update yaml file

* Support multiple images

* Update test

* Support copy images in parallel

* Update yaml

* Clean up

* Return err if no creds provided

* Fix tests

* Add err msg

* Add debug log

* Do not use CPE for targetImages

* Support platform

* Delete Jenkins specific creds

* Update groovy: do not handle Jenkins creds

* Delete unused code

* Fix: Support platform

* Fix: Support platform

* Apply suggestion from code review

Co-authored-by: Egor Balakin <14162703+m1ron0xFF@users.noreply.github.com>

* Apply suggestion from code review

Co-authored-by: Egor Balakin <14162703+m1ron0xFF@users.noreply.github.com>

* Add tests for parseDockerImageName

* Add comment that tagArtifactVersion is not supported yet

* Set limit of running goroutines

* Fix: Set limit of running goroutines

* The tagArtifactVersion is not supported yet

---------

Co-authored-by: Muhammadali Nazarov <muhammadalinazarov@gmail.com>
Co-authored-by: Egor Balakin <egor.balakin@sap.com>
Co-authored-by: Vyacheslav Starostin <vyacheslav.starostin@sap.com>
Co-authored-by: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com>
Co-authored-by: Egor Balakin <14162703+m1ron0xFF@users.noreply.github.com>
This commit is contained in:
Anil Keshav 2023-11-30 10:06:31 +01:00 committed by GitHub
parent cce7c0d384
commit 8dc2a1bfb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 1130 additions and 16 deletions

235
cmd/imagePushToRegistry.go Normal file
View File

@ -0,0 +1,235 @@
package cmd
import (
"context"
"fmt"
"regexp"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
"github.com/SAP/jenkins-library/pkg/command"
"github.com/SAP/jenkins-library/pkg/docker"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/telemetry"
)
const (
targetDockerConfigPath = "/root/.docker/config.json"
)
type dockerImageUtils interface {
LoadImage(ctx context.Context, src string) (v1.Image, error)
PushImage(ctx context.Context, im v1.Image, dest, platform string) error
CopyImage(ctx context.Context, src, dest, platform string) error
}
type imagePushToRegistryUtils interface {
command.ExecRunner
piperutils.FileUtils
dockerImageUtils
// Add more methods here, or embed additional interfaces, or remove/replace as required.
// The imagePushToRegistryUtils interface should be descriptive of your runtime dependencies,
// i.e. include everything you need to be able to mock in tests.
// Unit tests shall be executable in parallel (not depend on global state), and don't (re-)test dependencies.
}
type imagePushToRegistryUtilsBundle struct {
*command.Command
*piperutils.Files
dockerImageUtils
// Embed more structs as necessary to implement methods or interfaces you add to imagePushToRegistryUtils.
// Structs embedded in this way must each have a unique set of methods attached.
// If there is no struct which implements the method you need, attach the method to
// imagePushToRegistryUtilsBundle and forward to the implementation of the dependency.
}
func newImagePushToRegistryUtils() imagePushToRegistryUtils {
utils := imagePushToRegistryUtilsBundle{
Command: &command.Command{
StepName: "imagePushToRegistry",
},
Files: &piperutils.Files{},
dockerImageUtils: &docker.CraneUtilsBundle{},
}
// Reroute command output to logging framework
utils.Stdout(log.Writer())
utils.Stderr(log.Writer())
return &utils
}
func imagePushToRegistry(config imagePushToRegistryOptions, telemetryData *telemetry.CustomData) {
// Utils can be used wherever the command.ExecRunner interface is expected.
// It can also be used for example as a mavenExecRunner.
utils := newImagePushToRegistryUtils()
// For HTTP calls import piperhttp "github.com/SAP/jenkins-library/pkg/http"
// and use a &piperhttp.Client{} in a custom system
// Example: step checkmarxExecuteScan.go
// Error situations should be bubbled up until they reach the line below which will then stop execution
// through the log.Entry().Fatal() call leading to an os.Exit(1) in the end.
err := runImagePushToRegistry(&config, telemetryData, utils)
if err != nil {
log.Entry().WithError(err).Fatal("step execution failed")
}
}
func runImagePushToRegistry(config *imagePushToRegistryOptions, telemetryData *telemetry.CustomData, utils imagePushToRegistryUtils) error {
if len(config.TargetImages) == 0 {
config.TargetImages = config.SourceImages
}
if len(config.TargetImages) != len(config.SourceImages) {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.New("configuration error: please configure targetImage and sourceImage properly")
}
re := regexp.MustCompile(`^https?://`)
config.SourceRegistryURL = re.ReplaceAllString(config.SourceRegistryURL, "")
config.TargetRegistryURL = re.ReplaceAllString(config.TargetRegistryURL, "")
log.Entry().Debug("Handling destination registry credentials")
if err := handleCredentialsForPrivateRegistry(config.DockerConfigJSON, config.TargetRegistryURL, config.TargetRegistryUser, config.TargetRegistryPassword, utils); err != nil {
return errors.Wrap(err, "failed to handle credentials for target registry")
}
if len(config.LocalDockerImagePath) > 0 {
if err := pushLocalImageToTargetRegistry(config, utils); err != nil {
return errors.Wrapf(err, "failed to push local image to %q", config.TargetRegistryURL)
}
return nil
}
log.Entry().Debug("Handling source registry credentials")
if err := handleCredentialsForPrivateRegistry(config.DockerConfigJSON, config.SourceRegistryURL, config.SourceRegistryUser, config.SourceRegistryPassword, utils); err != nil {
return errors.Wrap(err, "failed to handle credentials for source registry")
}
if err := copyImages(config, utils); err != nil {
return errors.Wrap(err, "failed to copy images")
}
return nil
}
func handleCredentialsForPrivateRegistry(dockerConfigJsonPath, registry, username, password string, utils imagePushToRegistryUtils) error {
if len(dockerConfigJsonPath) == 0 {
if len(registry) == 0 || len(username) == 0 || len(password) == 0 {
return errors.New("docker credentials not provided")
}
if _, err := docker.CreateDockerConfigJSON(registry, username, password, "", targetDockerConfigPath, utils); err != nil {
return errors.Wrap(err, "failed to create new docker config")
}
return nil
}
if _, err := docker.CreateDockerConfigJSON(registry, username, password, targetDockerConfigPath, dockerConfigJsonPath, utils); err != nil {
return errors.Wrapf(err, "failed to update docker config %q", dockerConfigJsonPath)
}
if err := docker.MergeDockerConfigJSON(targetDockerConfigPath, dockerConfigJsonPath, utils); err != nil {
return errors.Wrapf(err, "failed to merge docker config files")
}
return nil
}
func copyImages(config *imagePushToRegistryOptions, utils imagePushToRegistryUtils) error {
g, ctx := errgroup.WithContext(context.Background())
g.SetLimit(10)
platform := config.TargetArchitecture
for i := 0; i < len(config.SourceImages); i++ {
src := fmt.Sprintf("%s/%s", config.SourceRegistryURL, config.SourceImages[i])
dst := fmt.Sprintf("%s/%s", config.TargetRegistryURL, config.TargetImages[i])
g.Go(func() error {
log.Entry().Infof("Copying %s to %s...", src, dst)
if err := utils.CopyImage(ctx, src, dst, platform); err != nil {
return err
}
log.Entry().Infof("Copying %s to %s... Done", src, dst)
return nil
})
if config.TagLatest {
g.Go(func() error {
// imageName is repository + image, e.g test.registry/testImage
imageName := parseDockerImageName(dst)
log.Entry().Infof("Copying %s to %s...", src, imageName)
if err := utils.CopyImage(ctx, src, imageName, platform); err != nil {
return err
}
log.Entry().Infof("Copying %s to %s... Done", src, imageName)
return nil
})
}
}
if err := g.Wait(); err != nil {
return err
}
return nil
}
func pushLocalImageToTargetRegistry(config *imagePushToRegistryOptions, utils imagePushToRegistryUtils) error {
g, ctx := errgroup.WithContext(context.Background())
g.SetLimit(10)
platform := config.TargetArchitecture
log.Entry().Infof("Loading local image...")
img, err := utils.LoadImage(ctx, config.LocalDockerImagePath)
if err != nil {
return err
}
log.Entry().Infof("Loading local image... Done")
for i := 0; i < len(config.TargetImages); i++ {
dst := fmt.Sprintf("%s/%s", config.TargetRegistryURL, config.TargetImages[i])
g.Go(func() error {
log.Entry().Infof("Pushing %s...", dst)
if err := utils.PushImage(ctx, img, dst, platform); err != nil {
return err
}
log.Entry().Infof("Pushing %s... Done", dst)
return nil
})
if config.TagLatest {
g.Go(func() error {
// imageName is repository + image, e.g test.registry/testImage
imageName := parseDockerImageName(dst)
log.Entry().Infof("Pushing %s...", imageName)
if err := utils.PushImage(ctx, img, imageName, platform); err != nil {
return err
}
log.Entry().Infof("Pushing %s... Done", imageName)
return nil
})
}
}
if err := g.Wait(); err != nil {
return err
}
return nil
}
func parseDockerImageName(image string) string {
re := regexp.MustCompile(`^(.*?)(?::([^:/]+))?$`)
matches := re.FindStringSubmatch(image)
if len(matches) > 1 {
return matches[1]
}
return image
}

View File

@ -0,0 +1,352 @@
// Code generated by piper's step-generator. DO NOT EDIT.
package cmd
import (
"fmt"
"os"
"time"
"github.com/SAP/jenkins-library/pkg/config"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/splunk"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/SAP/jenkins-library/pkg/validation"
"github.com/spf13/cobra"
)
type imagePushToRegistryOptions struct {
TargetImages []string `json:"targetImages,omitempty"`
SourceImages []string `json:"sourceImages,omitempty"`
SourceRegistryURL string `json:"sourceRegistryUrl,omitempty"`
SourceRegistryUser string `json:"sourceRegistryUser,omitempty"`
SourceRegistryPassword string `json:"sourceRegistryPassword,omitempty"`
TargetRegistryURL string `json:"targetRegistryUrl,omitempty"`
TargetRegistryUser string `json:"targetRegistryUser,omitempty"`
TargetRegistryPassword string `json:"targetRegistryPassword,omitempty"`
TagLatest bool `json:"tagLatest,omitempty"`
TagArtifactVersion bool `json:"tagArtifactVersion,omitempty"`
DockerConfigJSON string `json:"dockerConfigJSON,omitempty"`
LocalDockerImagePath string `json:"localDockerImagePath,omitempty"`
TargetArchitecture string `json:"targetArchitecture,omitempty"`
}
// ImagePushToRegistryCommand Allows you to copy a Docker image from a source container registry to a destination container registry.
func ImagePushToRegistryCommand() *cobra.Command {
const STEP_NAME = "imagePushToRegistry"
metadata := imagePushToRegistryMetadata()
var stepConfig imagePushToRegistryOptions
var startTime time.Time
var logCollector *log.CollectorHook
var splunkClient *splunk.Splunk
telemetryClient := &telemetry.Telemetry{}
var createImagePushToRegistryCmd = &cobra.Command{
Use: STEP_NAME,
Short: "Allows you to copy a Docker image from a source container registry to a destination container registry.",
Long: `In case you want to pull an existing image from a remote container registry, a source image and source registry needs to be specified.<br />
This makes it possible to move an image from one registry to another.
The imagePushToRegistry is not similar in functionality to containerPushToRegistry (which is currently a groovy based step and only be used in jenkins).
Currently the imagePushToRegistry only supports copying a local image or image from source remote registry to destination registry.`,
PreRunE: func(cmd *cobra.Command, _ []string) error {
startTime = time.Now()
log.SetStepName(STEP_NAME)
log.SetVerbose(GeneralConfig.Verbose)
GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens)
path, _ := os.Getwd()
fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path}
log.RegisterHook(fatalHook)
err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return err
}
log.RegisterSecret(stepConfig.SourceRegistryUser)
log.RegisterSecret(stepConfig.SourceRegistryPassword)
log.RegisterSecret(stepConfig.TargetRegistryUser)
log.RegisterSecret(stepConfig.TargetRegistryPassword)
log.RegisterSecret(stepConfig.DockerConfigJSON)
if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
log.RegisterHook(&sentryHook)
}
if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 || len(GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint) > 0 {
splunkClient = &splunk.Splunk{}
logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID}
log.RegisterHook(logCollector)
}
if err = log.RegisterANSHookIfConfigured(GeneralConfig.CorrelationID); err != nil {
log.Entry().WithError(err).Warn("failed to set up SAP Alert Notification Service log hook")
}
validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages())
if err != nil {
return err
}
if err = validation.ValidateStruct(stepConfig); err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return err
}
return nil
},
Run: func(_ *cobra.Command, _ []string) {
stepTelemetryData := telemetry.CustomData{}
stepTelemetryData.ErrorCode = "1"
handler := func() {
config.RemoveVaultSecretFiles()
stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
stepTelemetryData.ErrorCategory = log.GetErrorCategory().String()
stepTelemetryData.PiperCommitHash = GitCommit
telemetryClient.SetData(&stepTelemetryData)
telemetryClient.Send()
if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
splunkClient.Initialize(GeneralConfig.CorrelationID,
GeneralConfig.HookConfig.SplunkConfig.Dsn,
GeneralConfig.HookConfig.SplunkConfig.Token,
GeneralConfig.HookConfig.SplunkConfig.Index,
GeneralConfig.HookConfig.SplunkConfig.SendLogs)
splunkClient.Send(telemetryClient.GetData(), logCollector)
}
if len(GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint) > 0 {
splunkClient.Initialize(GeneralConfig.CorrelationID,
GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint,
GeneralConfig.HookConfig.SplunkConfig.ProdCriblToken,
GeneralConfig.HookConfig.SplunkConfig.ProdCriblIndex,
GeneralConfig.HookConfig.SplunkConfig.SendLogs)
splunkClient.Send(telemetryClient.GetData(), logCollector)
}
}
log.DeferExitHandler(handler)
defer handler()
telemetryClient.Initialize(GeneralConfig.NoTelemetry, STEP_NAME)
imagePushToRegistry(stepConfig, &stepTelemetryData)
stepTelemetryData.ErrorCode = "0"
log.Entry().Info("SUCCESS")
},
}
addImagePushToRegistryFlags(createImagePushToRegistryCmd, &stepConfig)
return createImagePushToRegistryCmd
}
func addImagePushToRegistryFlags(cmd *cobra.Command, stepConfig *imagePushToRegistryOptions) {
cmd.Flags().StringSliceVar(&stepConfig.TargetImages, "targetImages", []string{}, "Defines the names (incl. tag) of the images that will be pushed to the target registry. If empty, sourceImages will be used.\nPlease ensure that targetImages and sourceImages correspond to each other: the first image in sourceImages will be mapped to the first image in the targetImages parameter.\n")
cmd.Flags().StringSliceVar(&stepConfig.SourceImages, "sourceImages", []string{}, "Defines the names (incl. tag) of the images that will be pulled from source registry. This is helpful for moving images from one location to another.\nPlease ensure that targetImages and sourceImages correspond to each other: the first image in sourceImages will be mapped to the first image in the targetImages parameter.\n")
cmd.Flags().StringVar(&stepConfig.SourceRegistryURL, "sourceRegistryUrl", os.Getenv("PIPER_sourceRegistryUrl"), "Defines a registry url from where the image should optionally be pulled from, incl. the protocol like `https://my.registry.com`*\"")
cmd.Flags().StringVar(&stepConfig.SourceRegistryUser, "sourceRegistryUser", os.Getenv("PIPER_sourceRegistryUser"), "Username of the source registry where the image should be pushed pulled from.")
cmd.Flags().StringVar(&stepConfig.SourceRegistryPassword, "sourceRegistryPassword", os.Getenv("PIPER_sourceRegistryPassword"), "Password of the source registry where the image should be pushed pulled from.")
cmd.Flags().StringVar(&stepConfig.TargetRegistryURL, "targetRegistryUrl", os.Getenv("PIPER_targetRegistryUrl"), "Defines a registry url from where the image should optionally be pushed to, incl. the protocol like `https://my.registry.com`*\"")
cmd.Flags().StringVar(&stepConfig.TargetRegistryUser, "targetRegistryUser", os.Getenv("PIPER_targetRegistryUser"), "Username of the target registry where the image should be pushed to.")
cmd.Flags().StringVar(&stepConfig.TargetRegistryPassword, "targetRegistryPassword", os.Getenv("PIPER_targetRegistryPassword"), "Password of the target registry where the image should be pushed to.")
cmd.Flags().BoolVar(&stepConfig.TagLatest, "tagLatest", false, "Defines if the image should be tagged as `latest`")
cmd.Flags().BoolVar(&stepConfig.TagArtifactVersion, "tagArtifactVersion", false, "The parameter is not supported yet. Defines if the image should be tagged with the artifact version")
cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", os.Getenv("PIPER_dockerConfigJSON"), "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).")
cmd.Flags().StringVar(&stepConfig.LocalDockerImagePath, "localDockerImagePath", os.Getenv("PIPER_localDockerImagePath"), "If the `localDockerImagePath` is a directory, it will be read as an OCI image layout. Otherwise, `localDockerImagePath` is assumed to be a docker-style tarball.")
cmd.Flags().StringVar(&stepConfig.TargetArchitecture, "targetArchitecture", os.Getenv("PIPER_targetArchitecture"), "Specifies the targetArchitecture in the form os/arch[/variant][:osversion] (e.g. linux/amd64). All OS and architectures of the specified image will be copied if it is a multi-platform image. To only push a single platform to the target registry use this parameter")
cmd.MarkFlagRequired("sourceImages")
cmd.MarkFlagRequired("sourceRegistryUrl")
cmd.MarkFlagRequired("targetRegistryUrl")
cmd.MarkFlagRequired("targetRegistryUser")
cmd.MarkFlagRequired("targetRegistryPassword")
}
// retrieve step metadata
func imagePushToRegistryMetadata() config.StepData {
var theMetaData = config.StepData{
Metadata: config.StepMetadata{
Name: "imagePushToRegistry",
Aliases: []config.Alias{},
Description: "Allows you to copy a Docker image from a source container registry to a destination container registry.",
},
Spec: config.StepSpec{
Inputs: config.StepInputs{
Resources: []config.StepResources{
{Name: "source", Type: "stash"},
},
Parameters: []config.StepParameters{
{
Name: "targetImages",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "[]string",
Mandatory: false,
Aliases: []config.Alias{},
Default: []string{},
},
{
Name: "sourceImages",
ResourceRef: []config.ResourceReference{
{
Name: "commonPipelineEnvironment",
Param: "container/imageNameTags",
},
},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "[]string",
Mandatory: true,
Aliases: []config.Alias{},
Default: []string{},
},
{
Name: "sourceRegistryUrl",
ResourceRef: []config.ResourceReference{
{
Name: "commonPipelineEnvironment",
Param: "container/registryUrl",
},
},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: true,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_sourceRegistryUrl"),
},
{
Name: "sourceRegistryUser",
ResourceRef: []config.ResourceReference{
{
Name: "commonPipelineEnvironment",
Param: "container/repositoryUsername",
},
{
Name: "registryCredentialsVaultSecretName",
Type: "vaultSecret",
Default: "docker-registry",
},
},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_sourceRegistryUser"),
},
{
Name: "sourceRegistryPassword",
ResourceRef: []config.ResourceReference{
{
Name: "commonPipelineEnvironment",
Param: "container/repositoryPassword",
},
{
Name: "registryCredentialsVaultSecretName",
Type: "vaultSecret",
Default: "docker-registry",
},
},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_sourceRegistryPassword"),
},
{
Name: "targetRegistryUrl",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: true,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_targetRegistryUrl"),
},
{
Name: "targetRegistryUser",
ResourceRef: []config.ResourceReference{
{
Name: "registryCredentialsVaultSecretName",
Type: "vaultSecret",
Default: "docker-registry",
},
},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: true,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_targetRegistryUser"),
},
{
Name: "targetRegistryPassword",
ResourceRef: []config.ResourceReference{
{
Name: "registryCredentialsVaultSecretName",
Type: "vaultSecret",
Default: "docker-registry",
},
},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: true,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_targetRegistryPassword"),
},
{
Name: "tagLatest",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "bool",
Mandatory: false,
Aliases: []config.Alias{},
Default: false,
},
{
Name: "tagArtifactVersion",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "bool",
Mandatory: false,
Aliases: []config.Alias{},
Default: false,
},
{
Name: "dockerConfigJSON",
ResourceRef: []config.ResourceReference{
{
Name: "dockerConfigFileVaultSecretName",
Type: "vaultSecretFile",
Default: "docker-config",
},
},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_dockerConfigJSON"),
},
{
Name: "localDockerImagePath",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_localDockerImagePath"),
},
{
Name: "targetArchitecture",
ResourceRef: []config.ResourceReference{},
Scope: []string{"STEPS", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_targetArchitecture"),
},
},
},
Containers: []config.Container{
{Image: "gcr.io/go-containerregistry/crane:debug", EnvVars: []config.EnvVar{{Name: "container", Value: "docker"}}, Options: []config.Option{{Name: "-u", Value: "0"}, {Name: "--entrypoint", Value: ""}}},
},
},
}
return theMetaData
}

View File

@ -0,0 +1,20 @@
//go:build unit
// +build unit
package cmd
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestImagePushToRegistryCommand(t *testing.T) {
t.Parallel()
testCmd := ImagePushToRegistryCommand()
// only high level testing performed - details are tested in step generation procedure
assert.Equal(t, "imagePushToRegistry", testCmd.Use, "command name incorrect")
}

View File

@ -0,0 +1,240 @@
package cmd
import (
"testing"
"github.com/stretchr/testify/assert"
dockermock "github.com/SAP/jenkins-library/pkg/docker/mock"
"github.com/SAP/jenkins-library/pkg/mock"
)
const (
customDockerConfig = `{"auths":{"source.registry":{"auth":"c291cmNldXNlcjpzb3VyY2VwYXNzd29yZA=="},"target.registry":{"auth":"dGFyZ2V0dXNlcjp0YXJnZXRwYXNzd29yZA=="}}}`
dockerConfig = `{
"auths": {
"source.registry": {
"auth": "c291cmNldXNlcjpzb3VyY2VwYXNzd29yZA=="
},
"target.registry": {
"auth": "dGFyZ2V0dXNlcjp0YXJnZXRwYXNzd29yZA=="
},
"test.registry": {
"auth": "dGVzdHVzZXI6dGVzdHBhc3N3b3Jk"
}
}
}`
)
type imagePushToRegistryMockUtils struct {
*mock.ExecMockRunner
*mock.FilesMock
*dockermock.CraneMockUtils
}
func newImagePushToRegistryMockUtils(craneUtils *dockermock.CraneMockUtils) *imagePushToRegistryMockUtils {
utils := &imagePushToRegistryMockUtils{
ExecMockRunner: &mock.ExecMockRunner{},
FilesMock: &mock.FilesMock{},
CraneMockUtils: craneUtils,
}
return utils
}
func TestRunImagePushToRegistry(t *testing.T) {
t.Parallel()
t.Run("good case", func(t *testing.T) {
t.Parallel()
config := imagePushToRegistryOptions{
SourceRegistryURL: "https://source.registry",
SourceImages: []string{"source-image:latest"},
SourceRegistryUser: "sourceuser",
SourceRegistryPassword: "sourcepassword",
TargetRegistryURL: "https://target.registry",
TargetImages: []string{"target-image:latest"},
TargetRegistryUser: "targetuser",
TargetRegistryPassword: "targetpassword",
}
craneMockUtils := &dockermock.CraneMockUtils{}
utils := newImagePushToRegistryMockUtils(craneMockUtils)
err := runImagePushToRegistry(&config, nil, utils)
assert.NoError(t, err)
createdConfig, err := utils.FileRead(targetDockerConfigPath)
assert.NoError(t, err)
assert.Equal(t, customDockerConfig, string(createdConfig))
})
t.Run("failed to copy image", func(t *testing.T) {
t.Parallel()
config := imagePushToRegistryOptions{
SourceRegistryURL: "https://source.registry",
SourceRegistryUser: "sourceuser",
SourceRegistryPassword: "sourcepassword",
SourceImages: []string{"source-image:latest"},
TargetRegistryURL: "https://target.registry",
TargetRegistryUser: "targetuser",
TargetRegistryPassword: "targetpassword",
}
craneMockUtils := &dockermock.CraneMockUtils{
ErrCopyImage: dockermock.ErrCopyImage,
}
utils := newImagePushToRegistryMockUtils(craneMockUtils)
err := runImagePushToRegistry(&config, nil, utils)
assert.EqualError(t, err, "failed to copy images: copy image err")
})
t.Run("failed to push local image", func(t *testing.T) {
t.Parallel()
config := imagePushToRegistryOptions{
SourceRegistryURL: "https://source.registry",
SourceRegistryUser: "sourceuser",
SourceRegistryPassword: "sourcepassword",
SourceImages: []string{"source-image:latest"},
TargetRegistryURL: "https://target.registry",
TargetRegistryUser: "targetuser",
TargetRegistryPassword: "targetpassword",
LocalDockerImagePath: "/local/path",
}
craneMockUtils := &dockermock.CraneMockUtils{
ErrLoadImage: dockermock.ErrLoadImage,
}
utils := newImagePushToRegistryMockUtils(craneMockUtils)
err := runImagePushToRegistry(&config, nil, utils)
assert.EqualError(t, err, "failed to push local image to \"target.registry\": load image err")
})
}
func TestHandleCredentialsForPrivateRegistry(t *testing.T) {
t.Parallel()
craneMockUtils := &dockermock.CraneMockUtils{}
t.Run("no custom docker config provided", func(t *testing.T) {
t.Parallel()
utils := newImagePushToRegistryMockUtils(craneMockUtils)
utils.AddFile("targetDockerConfigPath", []byte("abc"))
err := handleCredentialsForPrivateRegistry("", "target.registry", "targetuser", "targetpassword", utils)
assert.NoError(t, err)
createdConfigFile, err := utils.FileRead(targetDockerConfigPath)
assert.NoError(t, err)
assert.Equal(t, `{"auths":{"target.registry":{"auth":"dGFyZ2V0dXNlcjp0YXJnZXRwYXNzd29yZA=="}}}`, string(createdConfigFile))
})
t.Run("custom docker config provided", func(t *testing.T) {
t.Parallel()
utils := newImagePushToRegistryMockUtils(craneMockUtils)
utils.AddFile(targetDockerConfigPath, []byte(customDockerConfig))
err := handleCredentialsForPrivateRegistry(targetDockerConfigPath, "test.registry", "testuser", "testpassword", utils)
assert.NoError(t, err)
createdConfigFile, err := utils.FileRead(targetDockerConfigPath)
assert.NoError(t, err)
assert.Equal(t, dockerConfig, string(createdConfigFile))
})
t.Run("wrong format of docker config", func(t *testing.T) {
t.Parallel()
utils := newImagePushToRegistryMockUtils(craneMockUtils)
utils.AddFile(targetDockerConfigPath, []byte(`{auths:}`))
err := handleCredentialsForPrivateRegistry("", "test.registry", "testuser", "testpassword", utils)
assert.EqualError(t, err, "failed to create new docker config: failed to unmarshal json file '/root/.docker/config.json': invalid character 'a' looking for beginning of object key string")
})
}
func TestPushLocalImageToTargetRegistry(t *testing.T) {
t.Parallel()
t.Run("good case", func(t *testing.T) {
t.Parallel()
craneMockUtils := &dockermock.CraneMockUtils{}
config := &imagePushToRegistryOptions{
LocalDockerImagePath: "/image/path",
TargetRegistryURL: "https://target.registry",
TagLatest: false,
}
utils := newImagePushToRegistryMockUtils(craneMockUtils)
err := pushLocalImageToTargetRegistry(config, utils)
assert.NoError(t, err)
})
t.Run("bad case - failed to load image", func(t *testing.T) {
t.Parallel()
craneMockUtils := &dockermock.CraneMockUtils{
ErrLoadImage: dockermock.ErrLoadImage,
}
config := &imagePushToRegistryOptions{
LocalDockerImagePath: "/image/path",
TargetRegistryURL: "https://target.registry",
TagLatest: false,
}
utils := newImagePushToRegistryMockUtils(craneMockUtils)
err := pushLocalImageToTargetRegistry(config, utils)
assert.EqualError(t, err, "load image err")
})
t.Run("bad case - failed to push image", func(t *testing.T) {
t.Parallel()
craneMockUtils := &dockermock.CraneMockUtils{
ErrPushImage: dockermock.ErrPushImage,
}
config := &imagePushToRegistryOptions{
LocalDockerImagePath: "/image/path",
TargetRegistryURL: "https://target.registry",
TargetImages: []string{"my-image:1.0.0"},
TagLatest: false,
}
utils := newImagePushToRegistryMockUtils(craneMockUtils)
err := pushLocalImageToTargetRegistry(config, utils)
assert.EqualError(t, err, "push image err")
})
}
func TestParseDockerImageName(t *testing.T) {
t.Parallel()
tests := []struct {
name, image, expected string
}{
{
name: "registry + imagename + tag",
image: "test.io/repo/test-image:1.0.0-12345",
expected: "test.io/repo/test-image",
},
{
name: "registry + imagename + tag (registry with port)",
image: "test.io:50000/repo/test-image:1.0.0-12345",
expected: "test.io:50000/repo/test-image",
},
{
name: "registry + imagename",
image: "test-test.io/repo/testimage",
expected: "test-test.io/repo/testimage",
},
{
name: "imagename + tag",
image: "testImage:1.0.0",
expected: "testImage",
},
{
name: "imagename",
image: "test-image",
expected: "test-image",
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
actual := parseDockerImageName(test.image)
assert.Equal(t, test.expected, actual)
})
}
}

View File

@ -72,6 +72,7 @@ func GetAllStepMetadata() map[string]config.StepData {
"gradleExecuteBuild": gradleExecuteBuildMetadata(),
"hadolintExecute": hadolintExecuteMetadata(),
"helmExecute": helmExecuteMetadata(),
"imagePushToRegistry": imagePushToRegistryMetadata(),
"influxWriteData": influxWriteDataMetadata(),
"integrationArtifactDeploy": integrationArtifactDeployMetadata(),
"integrationArtifactDownload": integrationArtifactDownloadMetadata(),

View File

@ -201,6 +201,7 @@ func Execute() {
rootCmd.AddCommand(TmsExportCommand())
rootCmd.AddCommand(IntegrationArtifactTransportCommand())
rootCmd.AddCommand(AscAppUploadCommand())
rootCmd.AddCommand(ImagePushToRegistryCommand())
addRootFlags(rootCmd)

View File

@ -0,0 +1,7 @@
# ${docGenStepName}
## ${docGenDescription}
## ${docGenParameters}
## ${docGenConfiguration}

View File

@ -122,6 +122,7 @@ nav:
- handlePipelineStepErrors: steps/handlePipelineStepErrors.md
- healthExecuteCheck: steps/healthExecuteCheck.md
- helmExecute: steps/helmExecute.md
- imagePushToRegistry: steps/imagePushToRegistry.md
- influxWriteData: steps/influxWriteData.md
- integrationArtifactDeploy: steps/integrationArtifactDeploy.md
- integrationArtifactDownload: steps/integrationArtifactDownload.md

40
pkg/docker/crane.go Normal file
View File

@ -0,0 +1,40 @@
package docker
import (
"context"
"github.com/google/go-containerregistry/pkg/crane"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
type CraneUtilsBundle struct{}
func (c *CraneUtilsBundle) CopyImage(ctx context.Context, src, dest, platform string) error {
p, err := parsePlatform(platform)
if err != nil {
return err
}
return crane.Copy(src, dest, crane.WithContext(ctx), crane.WithPlatform(p))
}
func (c *CraneUtilsBundle) PushImage(ctx context.Context, im v1.Image, dest, platform string) error {
p, err := parsePlatform(platform)
if err != nil {
return err
}
return crane.Push(im, dest, crane.WithContext(ctx), crane.WithPlatform(p))
}
func (c *CraneUtilsBundle) LoadImage(ctx context.Context, src string) (v1.Image, error) {
return crane.Load(src, crane.WithContext(ctx))
}
// parsePlatform is a wrapper for v1.ParsePlatform. It is necessary because
// v1.ParsePlatform returns an empty struct when the platform is equal to an empty string,
// whereas we expect 'nil'
func parsePlatform(p string) (*v1.Platform, error) {
if p == "" {
return nil, nil
}
return v1.ParsePlatform(p)
}

View File

@ -11,19 +11,18 @@ import (
"regexp"
"strings"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/pkg/errors"
"github.com/docker/cli/cli/config"
"github.com/docker/cli/cli/config/configfile"
cranecmd "github.com/google/go-containerregistry/cmd/crane/cmd"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/pkg/errors"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
)
// AuthEntry defines base64 encoded username:password required inside a Docker config.json
@ -93,9 +92,10 @@ func CreateDockerConfigJSON(registryURL, username, password, targetPath, configP
targetPath = configPath
}
dockerConfigContent := []byte{}
dockerConfig := map[string]interface{}{}
if exists, _ := utils.FileExists(configPath); exists {
dockerConfigContent, err := utils.FileRead(configPath)
if exists, err := utils.FileExists(configPath); exists {
dockerConfigContent, err = utils.FileRead(configPath)
if err != nil {
return "", fmt.Errorf("failed to read file '%v': %w", configPath, err)
}
@ -106,6 +106,13 @@ func CreateDockerConfigJSON(registryURL, username, password, targetPath, configP
}
}
if registryURL == "" || password == "" || username == "" {
if err := fileWrite(targetPath, dockerConfigContent, utils); err != nil {
return "", err
}
return targetPath, nil
}
credentialsBase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:%v", username, password)))
dockerAuth := AuthEntry{Auth: credentialsBase64}
@ -125,19 +132,26 @@ func CreateDockerConfigJSON(registryURL, username, password, targetPath, configP
return "", fmt.Errorf("failed to marshal Docker config.json: %w", err)
}
//always create the target path directories if any before writing
err = utils.MkdirAll(filepath.Dir(targetPath), 0777)
if err != nil {
return "", fmt.Errorf("failed to create directory path for the Docker config.json file %v:%w", targetPath, err)
}
err = utils.FileWrite(targetPath, jsonResult, 0666)
if err != nil {
return "", fmt.Errorf("failed to write Docker config.json: %w", err)
if err := fileWrite(targetPath, jsonResult, utils); err != nil {
return "", err
}
return targetPath, nil
}
func fileWrite(path string, content []byte, utils piperutils.FileUtils) error {
err := utils.MkdirAll(filepath.Dir(path), 0777)
if err != nil {
return fmt.Errorf("failed to create directory path for the Docker config.json file %v:%w", path, err)
}
err = utils.FileWrite(path, content, 0666)
if err != nil {
return fmt.Errorf("failed to write Docker config.json: %w", err)
}
return nil
}
// Client defines an docker client object
type Client struct {
imageName string
@ -289,7 +303,7 @@ func ImageListWithFilePath(imageName string, excludes []string, trimDir string,
for _, dockerfilePath := range matches {
// make sure that the path we have is relative
// ToDo: needs rework
//dockerfilePath = strings.ReplaceAll(dockerfilePath, cwd, ".")
// dockerfilePath = strings.ReplaceAll(dockerfilePath, cwd, ".")
if piperutils.ContainsString(excludes, dockerfilePath) {
log.Entry().Infof("Discard %v since it is in the exclude list %v", dockerfilePath, excludes)

30
pkg/docker/mock/crane.go Normal file
View File

@ -0,0 +1,30 @@
package mock
import (
"context"
"errors"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
var (
ErrCopyImage = errors.New("copy image err")
ErrPushImage = errors.New("push image err")
ErrLoadImage = errors.New("load image err")
)
type CraneMockUtils struct {
ErrCopyImage, ErrPushImage, ErrLoadImage error
}
func (c *CraneMockUtils) CopyImage(_ context.Context, src, dest, platform string) error {
return c.ErrCopyImage
}
func (c *CraneMockUtils) PushImage(_ context.Context, im v1.Image, dest, platform string) error {
return c.ErrPushImage
}
func (c *CraneMockUtils) LoadImage(_ context.Context, src string) (v1.Image, error) {
return nil, c.ErrLoadImage
}

View File

@ -0,0 +1,163 @@
metadata:
name: imagePushToRegistry
description: Allows you to copy a Docker image from a source container registry to a destination container registry.
longDescription: |-
In case you want to pull an existing image from a remote container registry, a source image and source registry needs to be specified.<br />
This makes it possible to move an image from one registry to another.
The imagePushToRegistry is not similar in functionality to containerPushToRegistry (which is currently a groovy based step and only be used in jenkins).
Currently the imagePushToRegistry only supports copying a local image or image from source remote registry to destination registry.
spec:
inputs:
resources:
- name: source
type: stash
params:
- name: targetImages
type: "[]string"
description: |
Defines the names (incl. tag) of the images that will be pushed to the target registry. If empty, sourceImages will be used.
Please ensure that targetImages and sourceImages correspond to each other: the first image in sourceImages will be mapped to the first image in the targetImages parameter.
scope:
- PARAMETERS
- STAGES
- STEPS
- name: sourceImages
type: "[]string"
description: |
Defines the names (incl. tag) of the images that will be pulled from source registry. This is helpful for moving images from one location to another.
Please ensure that targetImages and sourceImages correspond to each other: the first image in sourceImages will be mapped to the first image in the targetImages parameter.
mandatory: true
scope:
- PARAMETERS
- STAGES
- STEPS
resourceRef:
- name: commonPipelineEnvironment
param: container/imageNameTags
- name: sourceRegistryUrl
description: Defines a registry url from where the image should optionally be pulled from, incl. the protocol like `https://my.registry.com`*"
type: string
mandatory: true
scope:
- PARAMETERS
- STAGES
- STEPS
resourceRef:
- name: commonPipelineEnvironment
param: container/registryUrl
- name: sourceRegistryUser
type: string
secret: true
description: Username of the source registry where the image should be pushed pulled from.
scope:
- PARAMETERS
- STAGES
- STEPS
resourceRef:
- name: commonPipelineEnvironment
param: container/repositoryUsername
- type: vaultSecret
name: registryCredentialsVaultSecretName
default: docker-registry
- name: sourceRegistryPassword
type: string
secret: true
description: Password of the source registry where the image should be pushed pulled from.
scope:
- PARAMETERS
- STAGES
- STEPS
resourceRef:
- name: commonPipelineEnvironment
param: container/repositoryPassword
- type: vaultSecret
name: registryCredentialsVaultSecretName
default: docker-registry
- name: targetRegistryUrl
description: Defines a registry url from where the image should optionally be pushed to, incl. the protocol like `https://my.registry.com`*"
type: string
mandatory: true
scope:
- PARAMETERS
- STAGES
- STEPS
- name: targetRegistryUser
type: string
secret: true
mandatory: true
description: Username of the target registry where the image should be pushed to.
scope:
- PARAMETERS
- STAGES
- STEPS
resourceRef:
- type: vaultSecret
name: registryCredentialsVaultSecretName
default: docker-registry
- name: targetRegistryPassword
type: string
secret: true
description: Password of the target registry where the image should be pushed to.
mandatory: true
scope:
- PARAMETERS
- STAGES
- STEPS
resourceRef:
- type: vaultSecret
name: registryCredentialsVaultSecretName
default: docker-registry
- name: tagLatest
description: "Defines if the image should be tagged as `latest`"
type: bool
scope:
- PARAMETERS
- STAGES
- STEPS
- name: tagArtifactVersion
description: "The parameter is not supported yet. Defines if the image should be tagged with the artifact version"
type: bool
scope:
- PARAMETERS
- STAGES
- STEPS
- name: dockerConfigJSON
type: string
secret: true
description: Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).
scope:
- PARAMETERS
- STAGES
- STEPS
resourceRef:
- type: vaultSecretFile
name: dockerConfigFileVaultSecretName
default: docker-config
- name: localDockerImagePath
description: "If the `localDockerImagePath` is a directory, it will be read as an OCI image layout. Otherwise, `localDockerImagePath` is assumed to be a docker-style tarball."
type: string
scope:
- PARAMETERS
- STAGES
- STEPS
- name: targetArchitecture
type: string
description: Specifies the targetArchitecture in the form os/arch[/variant][:osversion] (e.g. linux/amd64). All OS and architectures of the specified image will be copied if it is a multi-platform image. To only push a single platform to the target registry use this parameter
scope:
- STEPS
- PARAMETERS
containers:
- image: gcr.io/go-containerregistry/crane:debug
command:
- /busybox/tail -f /dev/null
shell: /busybox/sh
options:
- name: -u
value: "0"
- name: --entrypoint
value: ""
env:
- name: container
value: docker

View File

@ -229,6 +229,7 @@ public class CommonStepsTest extends BasePiperTest{
'apiProviderList', //implementing new golang pattern without fields
'tmsUpload',
'tmsExport',
'imagePushToRegistry',
]
@Test

View File

@ -0,0 +1,9 @@
import groovy.transform.Field
@Field String STEP_NAME = getClass().getName()
@Field String METADATA_FILE = 'metadata/imagePushToRegistry.yaml'
void call(Map parameters = [:]) {
List credentials = []
piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
}