diff --git a/go.mod b/go.mod index 601891d4..db5639c7 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/fsnotify/fsnotify v1.4.9 github.com/go-training/helloworld v0.0.0-20200225145412-ba5f4379d78b github.com/google/go-cmp v0.5.4 - github.com/google/go-containerregistry v0.4.1-0.20210216200643-d81088d9983e + github.com/google/go-containerregistry v0.5.0 github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect github.com/mattmoor/dep-notify v0.0.0-20190205035814-a45dec370a17 github.com/onsi/gomega v1.10.3 // indirect diff --git a/go.sum b/go.sum index 871886ca..0b47f021 100644 --- a/go.sum +++ b/go.sum @@ -185,6 +185,8 @@ github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.4.1-0.20210216200643-d81088d9983e h1:8jPkeUOUhBGSDpuIUyjKHsC0I5/nI4zvmv6DB2TACh4= github.com/google/go-containerregistry v0.4.1-0.20210216200643-d81088d9983e/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= +github.com/google/go-containerregistry v0.5.0 h1:eb9sinv4PKm0AUwQGov0mvIdA4pyBGjRofxN4tWnMwM= +github.com/google/go-containerregistry v0.5.0/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/pkg/publish/daemon.go b/pkg/publish/daemon.go index 2bce7e15..67aa42ea 100644 --- a/pkg/publish/daemon.go +++ b/pkg/publish/daemon.go @@ -43,8 +43,13 @@ func NewDaemon(namer Namer, tags []string) Interface { return &demon{namer, tags} } +// getOpts returns daemon.Options. It's a var to allow it to be overridden during tests. +var getOpts = func(ctx context.Context) []daemon.Option { + return []daemon.Option{daemon.WithContext(ctx)} +} + // Publish implements publish.Interface -func (d *demon) Publish(_ context.Context, br build.Result, s string) (name.Reference, error) { +func (d *demon) Publish(ctx context.Context, br build.Result, s string) (name.Reference, error) { s = strings.TrimPrefix(s, build.StrictScheme) // https://github.com/google/go-containerregistry/issues/212 s = strings.ToLower(s) @@ -100,7 +105,7 @@ func (d *demon) Publish(_ context.Context, br build.Result, s string) (name.Refe } log.Printf("Loading %v", digestTag) - if _, err := daemon.Write(digestTag, img); err != nil { + if _, err := daemon.Write(digestTag, img, getOpts(ctx)...); err != nil { return nil, err } log.Printf("Loaded %v", digestTag) @@ -112,9 +117,7 @@ func (d *demon) Publish(_ context.Context, br build.Result, s string) (name.Refe return nil, err } - err = daemon.Tag(digestTag, tag) - - if err != nil { + if err := daemon.Tag(digestTag, tag, getOpts(ctx)...); err != nil { return nil, err } log.Printf("Added tag %v", tagName) diff --git a/pkg/publish/daemon_test.go b/pkg/publish/daemon_test.go index 55091d6d..f0f1c425 100644 --- a/pkg/publish/daemon_test.go +++ b/pkg/publish/daemon_test.go @@ -26,24 +26,30 @@ import ( "github.com/google/go-containerregistry/pkg/v1/random" ) -type MockImageLoader struct{} +type mockClient struct { + daemon.Client +} -var Tags []string - -func (m *MockImageLoader) ImageLoad(_ context.Context, _ io.Reader, _ bool) (types.ImageLoadResponse, error) { +func (m *mockClient) NegotiateAPIVersion(context.Context) {} +func (m *mockClient) ImageLoad(context.Context, io.Reader, bool) (types.ImageLoadResponse, error) { return types.ImageLoadResponse{ Body: ioutil.NopCloser(strings.NewReader("Loaded")), }, nil } -func (m *MockImageLoader) ImageTag(ctx context.Context, source, target string) error { - Tags = append(Tags, target) +func (m *mockClient) ImageTag(_ context.Context, _ string, tag string) error { + Tags = append(Tags, tag) return nil } +var Tags []string + func init() { - daemon.GetImageLoader = func() (daemon.ImageLoader, error) { - return &MockImageLoader{}, nil + getOpts = func(ctx context.Context) []daemon.Option { + return []daemon.Option{ + daemon.WithContext(ctx), + daemon.WithClient(&mockClient{}), + } } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/append.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/append.go index 89b0bb25..a4addd11 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/append.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/append.go @@ -15,10 +15,11 @@ package cmd import ( - "log" + "fmt" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/spf13/cobra" @@ -33,35 +34,44 @@ func NewCmdAppend(options *[]crane.Option) *cobra.Command { Use: "append", Short: "Append contents of a tarball to a remote image", Args: cobra.NoArgs, - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { var base v1.Image var err error if baseRef == "" { logs.Warn.Printf("base unspecified, using empty image") base = empty.Image - } else { base, err = crane.Pull(baseRef, *options...) if err != nil { - log.Fatalf("pulling %s: %v", baseRef, err) + return fmt.Errorf("pulling %s: %v", baseRef, err) } } img, err := crane.Append(base, newLayers...) if err != nil { - log.Fatalf("appending %v: %v", newLayers, err) + return fmt.Errorf("appending %v: %v", newLayers, err) } if outFile != "" { if err := crane.Save(img, newTag, outFile); err != nil { - log.Fatalf("writing output %q: %v", outFile, err) + return fmt.Errorf("writing output %q: %v", outFile, err) } } else { if err := crane.Push(img, newTag, *options...); err != nil { - log.Fatalf("pushing image %s: %v", newTag, err) + return fmt.Errorf("pushing image %s: %v", newTag, err) } + ref, err := name.ParseReference(newTag) + if err != nil { + return fmt.Errorf("parsing reference %s: %v", newTag, err) + } + d, err := img.Digest() + if err != nil { + return fmt.Errorf("digest: %v", err) + } + fmt.Println(ref.Context().Digest(d.String())) } + return nil }, } appendCmd.Flags().StringVarP(&baseRef, "base", "b", "", "Name of base image to append to") diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/auth.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/auth.go index 93ef6565..4c2ab2a4 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/auth.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/auth.go @@ -78,18 +78,18 @@ func NewCmdAuthGet(argv ...string) *cobra.Command { Short: "Implements a credential helper", Example: eg, Args: cobra.NoArgs, - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { b, err := ioutil.ReadAll(os.Stdin) if err != nil { - log.Fatal(err) + return err } reg, err := name.NewRegistry(strings.TrimSpace(string(b))) if err != nil { - log.Fatal(err) + return err } authorizer, err := authn.DefaultKeychain.Resolve(reg) if err != nil { - log.Fatal(err) + return err } // If we don't find any credentials, there's a magic error to return: @@ -103,15 +103,13 @@ func NewCmdAuthGet(argv ...string) *cobra.Command { auth, err := authorizer.Authorization() if err != nil { - log.Fatal(err) + return err } // Convert back to a form that credential helpers can parse so that this // can act as a meta credential helper. creds := toCreds(auth) - if err := json.NewEncoder(os.Stdout).Encode(creds); err != nil { - log.Fatal(err) - } + return json.NewEncoder(os.Stdout).Encode(creds) }, } } @@ -132,17 +130,15 @@ func NewCmdAuthLogin(argv ...string) *cobra.Command { Short: "Log in to a registry", Example: eg, Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { reg, err := name.NewRegistry(args[0]) if err != nil { - log.Fatal(err) + return err } opts.serverAddress = reg.Name() - if err := login(opts); err != nil { - log.Fatal(err) - } + return login(opts) }, } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/blob.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/blob.go index 7ddc6be3..ca32a983 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/blob.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/blob.go @@ -15,8 +15,8 @@ package cmd import ( + "fmt" "io" - "log" "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" @@ -29,19 +29,20 @@ func NewCmdBlob(options *[]crane.Option) *cobra.Command { Short: "Read a blob from the registry", Example: "crane blob ubuntu@sha256:4c1d20cdee96111c8acf1858b62655a37ce81ae48648993542b7ac363ac5c0e5 > blob.tar.gz", Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { src := args[0] layer, err := crane.PullLayer(src, *options...) if err != nil { - log.Fatalf("pulling layer %s: %v", src, err) + return fmt.Errorf("pulling layer %s: %v", src, err) } blob, err := layer.Compressed() if err != nil { - log.Fatalf("fetching blob %s: %v", src, err) + return fmt.Errorf("fetching blob %s: %v", src, err) } if _, err := io.Copy(cmd.OutOrStdout(), blob); err != nil { - log.Fatalf("copying blob %s: %v", src, err) + return fmt.Errorf("copying blob %s: %v", src, err) } + return nil }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/catalog.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/catalog.go index 2c82af57..f8eb7cf1 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/catalog.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/catalog.go @@ -16,7 +16,6 @@ package cmd import ( "fmt" - "log" "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" @@ -28,16 +27,17 @@ func NewCmdCatalog(options *[]crane.Option) *cobra.Command { Use: "catalog", Short: "List the repos in a registry", Args: cobra.ExactArgs(1), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { reg := args[0] repos, err := crane.Catalog(reg, *options...) if err != nil { - log.Fatalf("reading repos for %s: %v", reg, err) + return fmt.Errorf("reading repos for %s: %v", reg, err) } for _, repo := range repos { fmt.Println(repo) } + return nil }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/config.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/config.go index d2339c1c..4fa55162 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/config.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/config.go @@ -16,7 +16,6 @@ package cmd import ( "fmt" - "log" "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" @@ -28,12 +27,13 @@ func NewCmdConfig(options *[]crane.Option) *cobra.Command { Use: "config IMAGE", Short: "Get the config of an image", Args: cobra.ExactArgs(1), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { cfg, err := crane.Config(args[0], *options...) if err != nil { - log.Fatalf("fetching config: %v", err) + return fmt.Errorf("fetching config: %v", err) } fmt.Print(string(cfg)) + return nil }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/copy.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/copy.go index 993233a8..107d00b8 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/copy.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/copy.go @@ -15,8 +15,6 @@ package cmd import ( - "log" - "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" ) @@ -28,11 +26,9 @@ func NewCmdCopy(options *[]crane.Option) *cobra.Command { Aliases: []string{"cp"}, Short: "Efficiently copy a remote image from src to dst", Args: cobra.ExactArgs(2), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { src, dst := args[0], args[1] - if err := crane.Copy(src, dst, *options...); err != nil { - log.Fatal(err) - } + return crane.Copy(src, dst, *options...) }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/delete.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/delete.go index c5839abf..18966ea1 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/delete.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/delete.go @@ -15,8 +15,6 @@ package cmd import ( - "log" - "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" ) @@ -27,11 +25,9 @@ func NewCmdDelete(options *[]crane.Option) *cobra.Command { Use: "delete IMAGE", Short: "Delete an image reference from its registry", Args: cobra.ExactArgs(1), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { ref := args[0] - if err := crane.Delete(ref, *options...); err != nil { - log.Fatalf("deleting %s: %v", ref, err) - } + return crane.Delete(ref, *options...) }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/digest.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/digest.go index 24659e41..26e30218 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/digest.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/digest.go @@ -15,8 +15,8 @@ package cmd import ( + "errors" "fmt" - "log" "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" @@ -24,16 +24,52 @@ import ( // NewCmdDigest creates a new cobra.Command for the digest subcommand. func NewCmdDigest(options *[]crane.Option) *cobra.Command { - return &cobra.Command{ + var tarball string + cmd := &cobra.Command{ Use: "digest IMAGE", Short: "Get the digest of an image", - Args: cobra.ExactArgs(1), - Run: func(_ *cobra.Command, args []string) { - digest, err := crane.Digest(args[0], *options...) + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if tarball == "" && len(args) == 0 { + cmd.Help() + return errors.New("image reference required without --tarball") + } + + digest, err := getDigest(tarball, args, options) if err != nil { - log.Fatalf("computing digest: %v", err) + return err } fmt.Println(digest) + return nil }, } + + cmd.Flags().StringVar(&tarball, "tarball", "", "(Optional) path to tarball containing the image") + + return cmd +} + +func getDigest(tarball string, args []string, options *[]crane.Option) (string, error) { + if tarball != "" { + return getTarballDigest(tarball, args, options) + } + + return crane.Digest(args[0], *options...) +} + +func getTarballDigest(tarball string, args []string, options *[]crane.Option) (string, error) { + tag := "" + if len(args) > 0 { + tag = args[0] + } + + img, err := crane.LoadTag(tarball, tag, *options...) + if err != nil { + return "", fmt.Errorf("loading image from %q: %v", tarball, err) + } + digest, err := img.Digest() + if err != nil { + return "", fmt.Errorf("computing digest: %v", err) + } + return digest.String(), nil } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/export.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/export.go index 1c04da1b..268fa8ac 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/export.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/export.go @@ -15,7 +15,7 @@ package cmd import ( - "log" + "fmt" "os" "github.com/google/go-containerregistry/pkg/crane" @@ -33,23 +33,21 @@ func NewCmdExport(options *[]crane.Option) *cobra.Command { # Write tarball to file crane export ubuntu ubuntu.tar`, Args: cobra.ExactArgs(2), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { src, dst := args[0], args[1] f, err := openFile(dst) if err != nil { - log.Fatalf("failed to open %s: %v", dst, err) + return fmt.Errorf("failed to open %s: %v", dst, err) } defer f.Close() img, err := crane.Pull(src, *options...) if err != nil { - log.Fatal(err) + return fmt.Errorf("pulling %s: %v", src, err) } - if err := crane.Export(img, f); err != nil { - log.Fatalf("exporting %s: %v", src, err) - } + return crane.Export(img, f) }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/list.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/list.go index f8667dbd..0db1473b 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/list.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/list.go @@ -16,7 +16,6 @@ package cmd import ( "fmt" - "log" "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" @@ -28,16 +27,17 @@ func NewCmdList(options *[]crane.Option) *cobra.Command { Use: "ls REPO", Short: "List the tags in a repo", Args: cobra.ExactArgs(1), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { repo := args[0] tags, err := crane.ListTags(repo, *options...) if err != nil { - log.Fatalf("reading tags for %s: %v", repo, err) + return fmt.Errorf("reading tags for %s: %v", repo, err) } for _, tag := range tags { fmt.Println(tag) } + return nil }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/manifest.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/manifest.go index 35df558c..80d70b02 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/manifest.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/manifest.go @@ -16,7 +16,6 @@ package cmd import ( "fmt" - "log" "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" @@ -28,13 +27,14 @@ func NewCmdManifest(options *[]crane.Option) *cobra.Command { Use: "manifest IMAGE", Short: "Get the manifest of an image", Args: cobra.ExactArgs(1), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { src := args[0] manifest, err := crane.Manifest(src, *options...) if err != nil { - log.Fatalf("fetching manifest %s: %v", src, err) + return fmt.Errorf("fetching manifest %s: %v", src, err) } fmt.Print(string(manifest)) + return nil }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/mutate.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/mutate.go new file mode 100644 index 00000000..c444eaa4 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/mutate.go @@ -0,0 +1,107 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "log" + "strings" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/spf13/cobra" +) + +// NewCmdMutate creates a new cobra.Command for the mutate subcommand. +func NewCmdMutate(options *[]crane.Option) *cobra.Command { + var lbls []string + var entrypoint string + var newRef string + + mutateCmd := &cobra.Command{ + Use: "mutate", + Short: "Modify image labels and annotations", + Args: cobra.ExactArgs(1), + Run: func(_ *cobra.Command, args []string) { + // Pull image and get config. + ref := args[0] + img, err := crane.Pull(ref, *options...) + if err != nil { + log.Fatalf("pulling %s: %v", ref, err) + } + cfg, err := img.ConfigFile() + if err != nil { + log.Fatalf("getting config: %v", err) + } + cfg = cfg.DeepCopy() + + // Set labels. + if cfg.Config.Labels == nil { + cfg.Config.Labels = map[string]string{} + } + labels := map[string]string{} + for _, l := range lbls { + parts := strings.SplitN(l, "=", 2) + if len(parts) == 1 { + log.Fatalf("parsing label %q, not enough parts", l) + } + labels[parts[0]] = parts[1] + } + for k, v := range labels { + cfg.Config.Labels[k] = v + } + + // Set entrypoint. + if entrypoint != "" { + // NB: This doesn't attempt to do anything smart about splitting the string into multiple entrypoint elements. + cfg.Config.Entrypoint = []string{entrypoint} + } + + // Mutate and write image. + img, err = mutate.Config(img, cfg.Config) + if err != nil { + log.Fatalf("mutating config: %v", err) + } + + // If the new ref isn't provided, write over the original image. + // If that ref was provided by digest (e.g., output from + // another crane command), then strip that and push the + // mutated image by digest instead. + if newRef == "" { + newRef = ref + } + digest, err := img.Digest() + if err != nil { + log.Fatalf("digesting new image: %v", err) + } + r, err := name.ParseReference(newRef) + if err != nil { + log.Fatalf("parsing %s: %v", newRef, err) + } + if _, ok := r.(name.Digest); ok { + newRef = r.Context().Digest(digest.String()).String() + } + if err := crane.Push(img, newRef, *options...); err != nil { + log.Fatalf("pushing %s: %v", newRef, err) + } + fmt.Println(r.Context().Digest(digest.String())) + }, + } + mutateCmd.Flags().StringSliceVarP(&lbls, "label", "l", nil, "New labels to add") + mutateCmd.Flags().StringVar(&entrypoint, "entrypoint", "", "New entrypoing to set") + mutateCmd.Flags().StringVarP(&newRef, "tag", "t", "", "New tag to apply to mutated image. If not provided, push by digest to the original image repository.") + return mutateCmd +} diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/optimize.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/optimize.go index 214a0b4e..89c47066 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/optimize.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/optimize.go @@ -15,8 +15,6 @@ package cmd import ( - "log" - "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" ) @@ -31,11 +29,9 @@ func NewCmdOptimize(options *[]crane.Option) *cobra.Command { Aliases: []string{"opt"}, Short: "Optimize a remote container image from src to dst", Args: cobra.ExactArgs(2), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { src, dst := args[0], args[1] - if err := crane.Optimize(src, dst, files, *options...); err != nil { - log.Fatal(err) - } + return crane.Optimize(src, dst, files, *options...) }, } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/pull.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/pull.go index b5559926..08a1e938 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/pull.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/pull.go @@ -16,9 +16,9 @@ package cmd import ( "fmt" - "log" "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/cache" "github.com/spf13/cobra" ) @@ -29,34 +29,40 @@ func NewCmdPull(options *[]crane.Option) *cobra.Command { cmd := &cobra.Command{ Use: "pull IMAGE TARBALL", - Short: "Pull a remote image by reference and store its contents in a tarball", - Args: cobra.ExactArgs(2), - Run: func(_ *cobra.Command, args []string) { - src, path := args[0], args[1] - img, err := crane.Pull(src, *options...) - if err != nil { - log.Fatal(err) - } - if cachePath != "" { - img = cache.Image(img, cache.NewFilesystemCache(cachePath)) + Short: "Pull remote images by reference and store their contents in a tarball", + Args: cobra.MinimumNArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + imageMap := map[string]v1.Image{} + srcList, path := args[:len(args)-1], args[len(args)-1] + for _, src := range srcList { + img, err := crane.Pull(src, *options...) + if err != nil { + return fmt.Errorf("pulling %s: %v", src, err) + } + if cachePath != "" { + img = cache.Image(img, cache.NewFilesystemCache(cachePath)) + } + + imageMap[src] = img } switch format { case "tarball": - if err := crane.Save(img, src, path); err != nil { - log.Fatalf("saving tarball %s: %v", path, err) + if err := crane.MultiSave(imageMap, path); err != nil { + return fmt.Errorf("saving tarball %s: %v", path, err) } case "legacy": - if err := crane.SaveLegacy(img, src, path); err != nil { - log.Fatalf("saving legacy tarball %s: %v", path, err) + if err := crane.MultiSaveLegacy(imageMap, path); err != nil { + return fmt.Errorf("saving legacy tarball %s: %v", path, err) } case "oci": - if err := crane.SaveOCI(img, path); err != nil { - log.Fatalf("saving oci image layout %s: %v", path, err) + if err := crane.MultiSaveOCI(imageMap, path); err != nil { + return fmt.Errorf("saving oci image layout %s: %v", path, err) } default: - log.Fatalf("unexpected --format: %q (valid values are: tarball, legacy, and oci)", format) + return fmt.Errorf("unexpected --format: %q (valid values are: tarball, legacy, and oci)", format) } + return nil }, } cmd.Flags().StringVarP(&cachePath, "cache_path", "c", "", "Path to cache image layers") diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/push.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/push.go index fbdc6146..5a4731cd 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/push.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/push.go @@ -15,7 +15,7 @@ package cmd import ( - "log" + "fmt" "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" @@ -27,16 +27,14 @@ func NewCmdPush(options *[]crane.Option) *cobra.Command { Use: "push TARBALL IMAGE", Short: "Push image contents as a tarball to a remote registry", Args: cobra.ExactArgs(2), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { path, tag := args[0], args[1] img, err := crane.Load(path) if err != nil { - log.Fatalf("loading %s as tarball: %v", path, err) + return fmt.Errorf("loading %s as tarball: %v", path, err) } - if err := crane.Push(img, tag, *options...); err != nil { - log.Fatalf("pushing %s: %v", tag, err) - } + return crane.Push(img, tag, *options...) }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/rebase.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/rebase.go index 4226223f..8fe916fe 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/rebase.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/rebase.go @@ -16,7 +16,6 @@ package cmd import ( "fmt" - "log" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/v1/mutate" @@ -31,36 +30,37 @@ func NewCmdRebase(options *[]crane.Option) *cobra.Command { Use: "rebase", Short: "Rebase an image onto a new base image", Args: cobra.NoArgs, - Run: func(*cobra.Command, []string) { + RunE: func(*cobra.Command, []string) error { origImg, err := crane.Pull(orig, *options...) if err != nil { - log.Fatalf("pulling %s: %v", orig, err) + return fmt.Errorf("pulling %s: %v", orig, err) } oldBaseImg, err := crane.Pull(oldBase, *options...) if err != nil { - log.Fatalf("pulling %s: %v", oldBase, err) + return fmt.Errorf("pulling %s: %v", oldBase, err) } newBaseImg, err := crane.Pull(newBase, *options...) if err != nil { - log.Fatalf("pulling %s: %v", newBase, err) + return fmt.Errorf("pulling %s: %v", newBase, err) } img, err := mutate.Rebase(origImg, oldBaseImg, newBaseImg) if err != nil { - log.Fatalf("rebasing: %v", err) + return fmt.Errorf("rebasing: %v", err) } if err := crane.Push(img, rebased, *options...); err != nil { - log.Fatalf("pushing %s: %v", rebased, err) + return fmt.Errorf("pushing %s: %v", rebased, err) } digest, err := img.Digest() if err != nil { - log.Fatalf("digesting rebased: %v", err) + return fmt.Errorf("digesting rebased: %v", err) } fmt.Println(digest.String()) + return nil }, } rebaseCmd.Flags().StringVarP(&orig, "original", "", "", "Original image to rebase") diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/root.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/root.go index 75997281..f1123425 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/root.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/root.go @@ -93,6 +93,7 @@ func New(use, short string, options []crane.Option) *cobra.Command { NewCmdTag(&options), NewCmdValidate(&options), NewCmdVersion(), + NewCmdMutate(&options), } root.AddCommand(commands...) diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/tag.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/tag.go index 9e3338f7..9af803a6 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/tag.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/tag.go @@ -15,8 +15,6 @@ package cmd import ( - "log" - "github.com/google/go-containerregistry/pkg/crane" "github.com/spf13/cobra" ) @@ -38,11 +36,9 @@ crane tag registry.example.com/library/ubuntu:v0 v1 Example: `# Add a v1 tag to ubuntu crane tag ubuntu v1`, Args: cobra.ExactArgs(2), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { img, tag := args[0], args[1] - if err := crane.Tag(img, tag, *options...); err != nil { - log.Fatal(err) - } + return crane.Tag(img, tag, *options...) }, } } diff --git a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/validate.go b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/validate.go index c3edc88f..4494515e 100644 --- a/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/validate.go +++ b/vendor/github.com/google/go-containerregistry/cmd/crane/cmd/validate.go @@ -16,7 +16,6 @@ package cmd import ( "fmt" - "log" "github.com/google/go-containerregistry/pkg/crane" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -33,7 +32,7 @@ func NewCmdValidate(options *[]crane.Option) *cobra.Command { Use: "validate", Short: "Validate that an image is well-formed", Args: cobra.ExactArgs(0), - Run: func(_ *cobra.Command, args []string) { + RunE: func(_ *cobra.Command, args []string) error { for flag, maker := range map[string]func(string, ...crane.Option) (v1.Image, error){ tarballPath: makeTarball, remoteRef: crane.Pull, @@ -43,7 +42,7 @@ func NewCmdValidate(options *[]crane.Option) *cobra.Command { } img, err := maker(flag, *options...) if err != nil { - log.Fatalf("failed to read image %s: %v", flag, err) + return fmt.Errorf("failed to read image %s: %v", flag, err) } if err := validate.Image(img); err != nil { @@ -52,6 +51,7 @@ func NewCmdValidate(options *[]crane.Option) *cobra.Command { fmt.Printf("PASS: %s\n", flag) } } + return nil }, } validateCmd.Flags().StringVar(&tarballPath, "tarball", "", "Path to tarball to validate") diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/internal/and/and_closer.go b/vendor/github.com/google/go-containerregistry/internal/and/and_closer.go similarity index 100% rename from vendor/github.com/google/go-containerregistry/pkg/v1/internal/and/and_closer.go rename to vendor/github.com/google/go-containerregistry/internal/and/and_closer.go diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/internal/estargz/estargz.go b/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go similarity index 100% rename from vendor/github.com/google/go-containerregistry/pkg/v1/internal/estargz/estargz.go rename to vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/internal/gzip/zip.go b/vendor/github.com/google/go-containerregistry/internal/gzip/zip.go similarity index 98% rename from vendor/github.com/google/go-containerregistry/pkg/v1/internal/gzip/zip.go rename to vendor/github.com/google/go-containerregistry/internal/gzip/zip.go index 6f7bfd93..bf1d66a5 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/internal/gzip/zip.go +++ b/vendor/github.com/google/go-containerregistry/internal/gzip/zip.go @@ -20,7 +20,7 @@ import ( "compress/gzip" "io" - "github.com/google/go-containerregistry/pkg/v1/internal/and" + "github.com/google/go-containerregistry/internal/and" ) var gzipMagicHeader = []byte{'\x1f', '\x8b'} diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/httptest/httptest.go b/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go similarity index 100% rename from vendor/github.com/google/go-containerregistry/pkg/internal/httptest/httptest.go rename to vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go diff --git a/vendor/github.com/google/go-containerregistry/internal/legacy/copy.go b/vendor/github.com/google/go-containerregistry/internal/legacy/copy.go new file mode 100644 index 00000000..10467ba1 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/internal/legacy/copy.go @@ -0,0 +1,57 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package legacy provides methods for interacting with legacy image formats. +package legacy + +import ( + "bytes" + "encoding/json" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// CopySchema1 allows `[g]crane cp` to work with old images without adding +// full support for schema 1 images to this package. +func CopySchema1(desc *remote.Descriptor, srcRef, dstRef name.Reference, opts ...remote.Option) error { + m := schema1{} + if err := json.NewDecoder(bytes.NewReader(desc.Manifest)).Decode(&m); err != nil { + return err + } + + for _, layer := range m.FSLayers { + src := srcRef.Context().Digest(layer.BlobSum) + dst := dstRef.Context().Digest(layer.BlobSum) + + blob, err := remote.Layer(src, opts...) + if err != nil { + return err + } + + if err := remote.WriteLayer(dst.Context(), blob, opts...); err != nil { + return err + } + } + + return remote.Put(dstRef, desc, opts...) +} + +type fslayer struct { + BlobSum string `json:"blobSum"` +} + +type schema1 struct { + FSLayers []fslayer `json:"fsLayers"` +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/redact/redact.go b/vendor/github.com/google/go-containerregistry/internal/redact/redact.go similarity index 100% rename from vendor/github.com/google/go-containerregistry/pkg/internal/redact/redact.go rename to vendor/github.com/google/go-containerregistry/internal/redact/redact.go diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go b/vendor/github.com/google/go-containerregistry/internal/retry/retry.go similarity index 97% rename from vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go rename to vendor/github.com/google/go-containerregistry/internal/retry/retry.go index 542ba49b..133cb1c1 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go +++ b/vendor/github.com/google/go-containerregistry/internal/retry/retry.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/google/go-containerregistry/pkg/internal/retry/wait" + "github.com/google/go-containerregistry/internal/retry/wait" ) // Backoff is an alias of our own wait.Backoff to avoid name conflicts with diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/retry/wait/kubernetes_apimachinery_wait.go b/vendor/github.com/google/go-containerregistry/internal/retry/wait/kubernetes_apimachinery_wait.go similarity index 100% rename from vendor/github.com/google/go-containerregistry/pkg/internal/retry/wait/kubernetes_apimachinery_wait.go rename to vendor/github.com/google/go-containerregistry/internal/retry/wait/kubernetes_apimachinery_wait.go diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/internal/verify/verify.go b/vendor/github.com/google/go-containerregistry/internal/verify/verify.go similarity index 96% rename from vendor/github.com/google/go-containerregistry/pkg/v1/internal/verify/verify.go rename to vendor/github.com/google/go-containerregistry/internal/verify/verify.go index 366b6eef..d0bc8ac2 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/internal/verify/verify.go +++ b/vendor/github.com/google/go-containerregistry/internal/verify/verify.go @@ -20,8 +20,8 @@ import ( "hash" "io" + "github.com/google/go-containerregistry/internal/and" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/internal/and" ) type verifyReader struct { diff --git a/vendor/github.com/google/go-containerregistry/pkg/crane/append.go b/vendor/github.com/google/go-containerregistry/pkg/crane/append.go index 48f3bb6a..6d7253ab 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/crane/append.go +++ b/vendor/github.com/google/go-containerregistry/pkg/crane/append.go @@ -40,20 +40,33 @@ func Append(base v1.Image, paths ...string) (v1.Image, error) { } func getLayer(path string) (v1.Layer, error) { + f, err := streamFile(path) + if err != nil { + return nil, err + } + if f != nil { + return stream.NewLayer(f), nil + } + + return tarball.LayerFromFile(path) +} + +// If we're dealing with a named pipe, trying to open it multiple times will +// fail, so we need to do a streaming upload. +// +// returns nil, nil for non-streaming files +func streamFile(path string) (*os.File, error) { + if path == "-" { + return os.Stdin, nil + } fi, err := os.Stat(path) if err != nil { return nil, err } - // If we're dealing with a named pipe, trying to open it multiple times will - // fail, so we need to do a streaming upload. if !fi.Mode().IsRegular() { - rc, err := os.Open(path) - if err != nil { - return nil, err - } - return stream.NewLayer(rc), nil + return os.Open(path) } - return tarball.LayerFromFile(path) + return nil, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/crane/copy.go b/vendor/github.com/google/go-containerregistry/pkg/crane/copy.go index ab73cb36..b922b34a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/crane/copy.go +++ b/vendor/github.com/google/go-containerregistry/pkg/crane/copy.go @@ -17,8 +17,7 @@ package crane import ( "fmt" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/internal/legacy" + "github.com/google/go-containerregistry/internal/legacy" "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -59,7 +58,7 @@ func Copy(src, dst string, opt ...Option) error { } case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: // Handle schema 1 images separately. - if err := copySchema1(desc, srcRef, dstRef); err != nil { + if err := legacy.CopySchema1(desc, srcRef, dstRef, o.remote...); err != nil { return fmt.Errorf("failed to copy schema 1 image: %v", err) } default: @@ -87,16 +86,3 @@ func copyIndex(desc *remote.Descriptor, dstRef name.Reference, o options) error } return remote.WriteIndex(dstRef, idx, o.remote...) } - -func copySchema1(desc *remote.Descriptor, srcRef, dstRef name.Reference) error { - srcAuth, err := authn.DefaultKeychain.Resolve(srcRef.Context().Registry) - if err != nil { - return err - } - dstAuth, err := authn.DefaultKeychain.Resolve(dstRef.Context().Registry) - if err != nil { - return err - } - - return legacy.CopySchema1(desc, srcRef, dstRef, srcAuth, dstAuth) -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/crane/digest.go b/vendor/github.com/google/go-containerregistry/pkg/crane/digest.go index 7fa774b6..697cc4af 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/crane/digest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/crane/digest.go @@ -14,6 +14,8 @@ package crane +import "github.com/google/go-containerregistry/pkg/logs" + // Digest returns the sha256 hash of the remote image at ref. func Digest(ref string, opt ...Option) (string, error) { o := makeOptions(opt...) @@ -39,7 +41,12 @@ func Digest(ref string, opt ...Option) (string, error) { } desc, err := head(ref, opt...) if err != nil { - return "", err + logs.Warn.Printf("HEAD request failed, falling back on GET: %v", err) + rdesc, err := getManifest(ref, opt...) + if err != nil { + return "", err + } + return rdesc.Digest.String(), nil } return desc.Digest.String(), nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/crane/pull.go b/vendor/github.com/google/go-containerregistry/pkg/crane/pull.go index 74beaa1a..124cfbdb 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/crane/pull.go +++ b/vendor/github.com/google/go-containerregistry/pkg/crane/pull.go @@ -37,7 +37,7 @@ func Pull(src string, opt ...Option) (v1.Image, error) { o := makeOptions(opt...) ref, err := name.ParseReference(src, o.name...) if err != nil { - return nil, fmt.Errorf("parsing tag %q: %v", src, err) + return nil, fmt.Errorf("parsing reference %q: %v", src, err) } return remote.Image(ref, o.remote...) @@ -45,26 +45,36 @@ func Pull(src string, opt ...Option) (v1.Image, error) { // Save writes the v1.Image img as a tarball at path with tag src. func Save(img v1.Image, src, path string) error { - ref, err := name.ParseReference(src) - if err != nil { - return fmt.Errorf("parsing ref %q: %v", src, err) - } + imgMap := map[string]v1.Image{src: img} + return MultiSave(imgMap, path) +} - // WriteToFile wants a tag to write to the tarball, but we might have - // been given a digest. - // If the original ref was a tag, use that. Otherwise, if it was a - // digest, tag the image with :i-was-a-digest instead. - tag, ok := ref.(name.Tag) - if !ok { - d, ok := ref.(name.Digest) - if !ok { - return fmt.Errorf("ref wasn't a tag or digest") +// MultiSave writes collection of v1.Image img with tag as a tarball. +func MultiSave(imgMap map[string]v1.Image, path string) error { + tagToImage := map[name.Tag]v1.Image{} + + for src, img := range imgMap { + ref, err := name.ParseReference(src) + if err != nil { + return fmt.Errorf("parsing ref %q: %v", src, err) } - tag = d.Repository.Tag(iWasADigestTag) - } + // WriteToFile wants a tag to write to the tarball, but we might have + // been given a digest. + // If the original ref was a tag, use that. Otherwise, if it was a + // digest, tag the image with :i-was-a-digest instead. + tag, ok := ref.(name.Tag) + if !ok { + d, ok := ref.(name.Digest) + if !ok { + return fmt.Errorf("ref wasn't a tag or digest") + } + tag = d.Repository.Tag(iWasADigestTag) + } + tagToImage[tag] = img + } // no progress channel (for now) - return tarball.WriteToFile(path, tag, img) + return tarball.MultiWriteToFile(path, tagToImage) } // PullLayer returns the given layer from a registry. @@ -80,9 +90,20 @@ func PullLayer(ref string, opt ...Option) (v1.Layer, error) { // SaveLegacy writes the v1.Image img as a legacy tarball at path with tag src. func SaveLegacy(img v1.Image, src, path string) error { - ref, err := name.ParseReference(src) - if err != nil { - return fmt.Errorf("parsing ref %q: %v", src, err) + imgMap := map[string]v1.Image{src: img} + return MultiSave(imgMap, path) +} + +// MultiSaveLegacy writes collection of v1.Image img with tag as a legacy tarball. +func MultiSaveLegacy(imgMap map[string]v1.Image, path string) error { + refToImage := map[name.Reference]v1.Image{} + + for src, img := range imgMap { + ref, err := name.ParseReference(src) + if err != nil { + return fmt.Errorf("parsing ref %q: %v", src, err) + } + refToImage[ref] = img } w, err := os.Create(path) @@ -91,12 +112,19 @@ func SaveLegacy(img v1.Image, src, path string) error { } defer w.Close() - return legacy.Write(ref, img, w) + return legacy.MultiWrite(refToImage, w) } // SaveOCI writes the v1.Image img as an OCI Image Layout at path. If a layout // already exists at that path, it will add the image to the index. func SaveOCI(img v1.Image, path string) error { + imgMap := map[string]v1.Image{"": img} + return MultiSaveOCI(imgMap, path) +} + +// MultiSaveOCI writes collection of v1.Image img as an OCI Image Layout at path. If a layout +// already exists at that path, it will add the image to the index. +func MultiSaveOCI(imgMap map[string]v1.Image, path string) error { p, err := layout.FromPath(path) if err != nil { p, err = layout.Write(path, empty.Index) @@ -104,5 +132,10 @@ func SaveOCI(img v1.Image, path string) error { return err } } - return p.AppendImage(img) + for _, img := range imgMap { + if err = p.AppendImage(img); err != nil { + return err + } + } + return nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/crane/push.go b/vendor/github.com/google/go-containerregistry/pkg/crane/push.go index 52f4fc48..65dee4f8 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/crane/push.go +++ b/vendor/github.com/google/go-containerregistry/pkg/crane/push.go @@ -24,17 +24,31 @@ import ( ) // Load reads the tarball at path as a v1.Image. -func Load(path string) (v1.Image, error) { - // TODO: Allow tag? - return tarball.ImageFromPath(path, nil) +func Load(path string, opt ...Option) (v1.Image, error) { + return LoadTag(path, "") +} + +// LoadTag reads a tag from the tarball at path as a v1.Image. +// If tag is "", will attempt to read the tarball as a single image. +func LoadTag(path, tag string, opt ...Option) (v1.Image, error) { + if tag == "" { + return tarball.ImageFromPath(path, nil) + } + + o := makeOptions(opt...) + t, err := name.NewTag(tag, o.name...) + if err != nil { + return nil, fmt.Errorf("parsing tag %q: %v", tag, err) + } + return tarball.ImageFromPath(path, &t) } // Push pushes the v1.Image img to a registry as dst. func Push(img v1.Image, dst string, opt ...Option) error { o := makeOptions(opt...) - tag, err := name.NewTag(dst, o.name...) + tag, err := name.ParseReference(dst, o.name...) if err != nil { - return fmt.Errorf("parsing tag %q: %v", dst, err) + return fmt.Errorf("parsing reference %q: %v", dst, err) } return remote.Write(tag, img, o.remote...) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/legacy/copy.go b/vendor/github.com/google/go-containerregistry/pkg/internal/legacy/copy.go deleted file mode 100644 index 557c1d1f..00000000 --- a/vendor/github.com/google/go-containerregistry/pkg/internal/legacy/copy.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2019 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacy provides methods for interacting with legacy image formats. -package legacy - -import ( - "bytes" - "encoding/json" - "fmt" - "net/http" - "net/url" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/logs" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" -) - -// CopySchema1 allows `[g]crane cp` to work with old images without adding -// full support for schema 1 images to this package. -func CopySchema1(desc *remote.Descriptor, srcRef, dstRef name.Reference, srcAuth, dstAuth authn.Authenticator) error { - m := schema1{} - if err := json.NewDecoder(bytes.NewReader(desc.Manifest)).Decode(&m); err != nil { - return err - } - - for _, layer := range m.FSLayers { - src := srcRef.Context().Digest(layer.BlobSum) - dst := dstRef.Context().Digest(layer.BlobSum) - - blob, err := remote.Layer(src, remote.WithAuth(srcAuth)) - if err != nil { - return err - } - - if err := remote.WriteLayer(dst.Context(), blob, remote.WithAuth(dstAuth)); err != nil { - return err - } - } - - return putManifest(desc, dstRef, dstAuth) -} - -// TODO: perhaps expose this in remote? -func putManifest(desc *remote.Descriptor, dstRef name.Reference, dstAuth authn.Authenticator) error { - reg := dstRef.Context().Registry - scopes := []string{dstRef.Scope(transport.PushScope)} - - // TODO(jonjohnsonjr): Use NewWithContext. - tr, err := transport.New(reg, dstAuth, http.DefaultTransport, scopes) - if err != nil { - return err - } - client := &http.Client{Transport: tr} - - u := url.URL{ - Scheme: dstRef.Context().Registry.Scheme(), - Host: dstRef.Context().RegistryStr(), - Path: fmt.Sprintf("/v2/%s/manifests/%s", dstRef.Context().RepositoryStr(), dstRef.Identifier()), - } - - req, err := http.NewRequest(http.MethodPut, u.String(), bytes.NewBuffer(desc.Manifest)) - if err != nil { - return err - } - req.Header.Set("Content-Type", string(desc.MediaType)) - - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if err := transport.CheckError(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted); err != nil { - return err - } - - // The image was successfully pushed! - logs.Progress.Printf("%v: digest: %v size: %d", dstRef, desc.Digest, len(desc.Manifest)) - return nil -} - -type fslayer struct { - BlobSum string `json:"blobSum"` -} - -type schema1 struct { - FSLayers []fslayer `json:"fsLayers"` -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go index f9388253..e5180b3d 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go @@ -47,3 +47,30 @@ func ParseReference(s string, opts ...Option) (Reference, error) { return nil, NewErrBadName("could not parse reference: " + s) } + +type stringConst string + +// MustParseReference behaves like ParseReference, but panics instead of +// returning an error. It's intended for use in tests, or when a value is +// expected to be valid at code authoring time. +// +// To discourage its use in scenarios where the value is not known at code +// authoring time, it must be passed a string constant: +// +// const str = "valid/string" +// MustParseReference(str) +// MustParseReference("another/valid/string") +// MustParseReference(str + "/and/more") +// +// These will not compile: +// +// var str = "valid/string" +// MustParseReference(str) +// MustParseReference(strings.Join([]string{"valid", "string"}, "/")) +func MustParseReference(s stringConst, opts ...Option) Reference { + ref, err := ParseReference(string(s), opts...) + if err != nil { + panic(err) + } + return ref +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go b/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go index 5dc920ab..5bb4fe89 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" "sort" "strconv" @@ -31,6 +32,10 @@ import ( "github.com/google/go-containerregistry/pkg/v1/types" ) +type catalog struct { + Repos []string `json:"repositories"` +} + type listTags struct { Name string `json:"name"` Tags []string `json:"tags"` @@ -45,6 +50,7 @@ type manifests struct { // maps repo -> manifest tag/digest -> manifest manifests map[string]map[string]manifest lock sync.Mutex + log *log.Logger } func isManifest(req *http.Request) bool { @@ -65,6 +71,16 @@ func isTags(req *http.Request) bool { return elems[len(elems)-2] == "tags" } +func isCatalog(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 2 { + return false + } + + return elems[len(elems)-1] == "_catalog" +} + // https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-an-image-manifest // https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-an-image func (m *manifests) handle(resp http.ResponseWriter, req *http.Request) *regError { @@ -172,6 +188,7 @@ func (m *manifests) handle(resp http.ResponseWriter, req *http.Request) *regErro } } else { // TODO: Probably want to do an existence check for blobs. + m.log.Printf("TODO: Check blobs for %q", desc.Digest) } } } @@ -273,3 +290,45 @@ func (m *manifests) handleTags(resp http.ResponseWriter, req *http.Request) *reg Message: "We don't understand your method + url", } } + +func (m *manifests) handleCatalog(resp http.ResponseWriter, req *http.Request) *regError { + query := req.URL.Query() + nStr := query.Get("n") + n := 10000 + if nStr != "" { + n, _ = strconv.Atoi(nStr) + } + + if req.Method == "GET" { + m.lock.Lock() + defer m.lock.Unlock() + + var repos []string + countRepos := 0 + // TODO: implement pagination + for key := range m.manifests { + if countRepos >= n { + break + } + countRepos++ + + repos = append(repos, key) + } + + repositoriesToList := catalog{ + Repos: repos, + } + + msg, _ := json.Marshal(repositoriesToList) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go b/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go index 8e92bfb2..c56dae26 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go @@ -47,6 +47,9 @@ func (r *registry) v2(resp http.ResponseWriter, req *http.Request) *regError { if isTags(req) { return r.manifests.handleTags(resp, req) } + if isCatalog(req) { + return r.manifests.handleCatalog(resp, req) + } resp.Header().Set("Docker-Distribution-API-Version", "registry/2.0") if req.URL.Path != "/v2/" && req.URL.Path != "/v2" { return ®Error{ @@ -79,6 +82,7 @@ func New(opts ...Option) http.Handler { }, manifests: manifests{ manifests: map[string]map[string]manifest{}, + log: log.New(os.Stderr, "", log.LstdFlags), }, } for _, o := range opts { @@ -95,5 +99,6 @@ type Option func(r *registry) func Logger(l *log.Logger) Option { return func(r *registry) { r.log = l + r.manifests.log = l } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go b/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go index d5e86192..cb2644e6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go @@ -17,7 +17,7 @@ package registry import ( "net/http/httptest" - ggcrtest "github.com/google/go-containerregistry/pkg/internal/httptest" + ggcrtest "github.com/google/go-containerregistry/internal/httptest" ) // TLS returns an httptest server, with an http client that has been configured to diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/cache/cache.go b/vendor/github.com/google/go-containerregistry/pkg/v1/cache/cache.go index dd0cdebd..54a0dc43 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/cache/cache.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/cache/cache.go @@ -3,9 +3,11 @@ package cache import ( "errors" + "io" "github.com/google/go-containerregistry/pkg/logs" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" ) // Cache encapsulates methods to interact with cached layers. @@ -55,46 +57,66 @@ func (i *image) Layers() ([]v1.Layer, error) { var out []v1.Layer for _, l := range ls { - // Check if this layer is present in the cache in compressed - // form. - digest, err := l.Digest() - if err != nil { - return nil, err - } - if cl, err := i.c.Get(digest); err == nil { - // Layer found in the cache. - logs.Progress.Printf("Layer %s found (compressed) in cache", digest) - out = append(out, cl) - continue - } else if err != nil && err != ErrNotFound { - return nil, err - } - - // Check if this layer is present in the cache in - // uncompressed form. - diffID, err := l.DiffID() - if err != nil { - return nil, err - } - if cl, err := i.c.Get(diffID); err == nil { - // Layer found in the cache. - logs.Progress.Printf("Layer %s found (uncompressed) in cache", diffID) - out = append(out, cl) - } else if err != nil && err != ErrNotFound { - return nil, err - } - - // Not cached, fall through to real layer. - l, err = i.c.Put(l) - if err != nil { - return nil, err - } - out = append(out, l) - + out = append(out, &lazyLayer{inner: l, c: i.c}) } return out, nil } +type lazyLayer struct { + inner v1.Layer + c Cache +} + +func (l *lazyLayer) Compressed() (io.ReadCloser, error) { + digest, err := l.inner.Digest() + if err != nil { + return nil, err + } + + if cl, err := l.c.Get(digest); err == nil { + // Layer found in the cache. + logs.Progress.Printf("Layer %s found (compressed) in cache", digest) + return cl.Compressed() + } else if err != nil && err != ErrNotFound { + return nil, err + } + + // Not cached, pull and return the real layer. + logs.Progress.Printf("Layer %s not found (compressed) in cache, getting", digest) + rl, err := l.c.Put(l.inner) + if err != nil { + return nil, err + } + return rl.Compressed() +} + +func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) { + diffID, err := l.inner.DiffID() + if err != nil { + return nil, err + } + if cl, err := l.c.Get(diffID); err == nil { + // Layer found in the cache. + logs.Progress.Printf("Layer %s found (uncompressed) in cache", diffID) + return cl.Uncompressed() + } else if err != nil && err != ErrNotFound { + return nil, err + } + + // Not cached, pull and return the real layer. + logs.Progress.Printf("Layer %s not found (uncompressed) in cache, getting", diffID) + rl, err := l.c.Put(l.inner) + if err != nil { + return nil, err + } + return rl.Uncompressed() +} + +func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() } +func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.DiffID() } +func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() } +func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() } + func (i *image) LayerByDigest(h v1.Hash) (v1.Layer, error) { l, err := i.c.Get(h) if err == ErrNotFound { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go index eb7a9d7f..5013c3d3 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go @@ -19,103 +19,71 @@ import ( "context" "io" "io/ioutil" + "sync" - "github.com/docker/docker/client" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/tarball" ) -// image accesses an image from a docker daemon -type image struct { - v1.Image -} - -var _ v1.Image = (*image)(nil) - type imageOpener struct { - ref name.Reference + ref name.Reference + ctx context.Context + buffered bool client Client + + once sync.Once + bytes []byte + err error } -// ImageOption is a functional option for Image. -type ImageOption func(*imageOpener) error - -func (i *imageOpener) Open() (v1.Image, error) { - var opener tarball.Opener - var err error - if i.buffered { - opener, err = i.bufferedOpener(i.ref) - } else { - opener, err = i.unbufferedOpener(i.ref) - } - if err != nil { - return nil, err - } - - tb, err := tarball.Image(opener, nil) - if err != nil { - return nil, err - } - img := &image{ - Image: tb, - } - return img, nil +func (i *imageOpener) saveImage() (io.ReadCloser, error) { + return i.client.ImageSave(i.ctx, []string{i.ref.Name()}) } -func (i *imageOpener) saveImage(ref name.Reference) (io.ReadCloser, error) { - return i.client.ImageSave(context.Background(), []string{ref.Name()}) -} - -func (i *imageOpener) bufferedOpener(ref name.Reference) (tarball.Opener, error) { +func (i *imageOpener) bufferedOpener() (io.ReadCloser, error) { // Store the tarball in memory and return a new reader into the bytes each time we need to access something. - rc, err := i.saveImage(ref) - if err != nil { - return nil, err - } - defer rc.Close() + i.once.Do(func() { + i.bytes, i.err = func() ([]byte, error) { + rc, err := i.saveImage() + if err != nil { + return nil, err + } + defer rc.Close() - imageBytes, err := ioutil.ReadAll(rc) - if err != nil { - return nil, err - } - // The tarball interface takes a function that it can call to return an opened reader-like object. - // Daemon comes from a set of bytes, so wrap them in a ReadCloser so it looks like an opened file. - return func() (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(imageBytes)), nil - }, nil + return ioutil.ReadAll(rc) + }() + }) + + // Wrap the bytes in a ReadCloser so it looks like an opened file. + return ioutil.NopCloser(bytes.NewReader(i.bytes)), i.err } -func (i *imageOpener) unbufferedOpener(ref name.Reference) (tarball.Opener, error) { +func (i *imageOpener) opener() tarball.Opener { + if i.buffered { + return i.bufferedOpener + } + // To avoid storing the tarball in memory, do a save every time we need to access something. - return func() (io.ReadCloser, error) { - return i.saveImage(ref) - }, nil + return i.saveImage } // Image provides access to an image reference from the Docker daemon, // applying functional options to the underlying imageOpener before // resolving the reference into a v1.Image. -func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { +func Image(ref name.Reference, options ...Option) (v1.Image, error) { + o, err := makeOptions(options...) + if err != nil { + return nil, err + } + i := &imageOpener{ ref: ref, - buffered: true, // buffer by default - } - for _, option := range options { - if err := option(i); err != nil { - return nil, err - } + buffered: o.buffered, + client: o.client, + ctx: o.ctx, } - if i.client == nil { - var err error - i.client, err = client.NewClientWithOpts(client.FromEnv) - if err != nil { - return nil, err - } - } - i.client.NegotiateAPIVersion(context.Background()) - - return i.Open() + return tarball.Image(i.opener(), nil) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go index bb6726bc..c3a0ac66 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go @@ -19,34 +19,76 @@ import ( "io" "github.com/docker/docker/api/types" + "github.com/docker/docker/client" ) +// ImageOption is an alias for Option. +// Deprecated: Use Option instead. +type ImageOption Option + +// Option is a functional option for daemon operations. +type Option func(*options) + +type options struct { + ctx context.Context + client Client + buffered bool +} + +var defaultClient = func() (Client, error) { + return client.NewClientWithOpts(client.FromEnv) +} + +func makeOptions(opts ...Option) (*options, error) { + o := &options{ + buffered: true, + ctx: context.Background(), + } + for _, opt := range opts { + opt(o) + } + + if o.client == nil { + client, err := defaultClient() + if err != nil { + return nil, err + } + o.client = client + } + o.client.NegotiateAPIVersion(o.ctx) + + return o, nil +} + // WithBufferedOpener buffers the image. -func WithBufferedOpener() ImageOption { - return func(i *imageOpener) error { - return i.setBuffered(true) +func WithBufferedOpener() Option { + return func(o *options) { + o.buffered = true } } // WithUnbufferedOpener streams the image to avoid buffering. -func WithUnbufferedOpener() ImageOption { - return func(i *imageOpener) error { - return i.setBuffered(false) +func WithUnbufferedOpener() Option { + return func(o *options) { + o.buffered = false } } -func (i *imageOpener) setBuffered(buffer bool) error { - i.buffered = buffer - return nil -} - // WithClient is a functional option to allow injecting a docker client. // // By default, github.com/docker/docker/client.FromEnv is used. -func WithClient(client Client) ImageOption { - return func(i *imageOpener) error { - i.client = client - return nil +func WithClient(client Client) Option { + return func(o *options) { + o.client = client + } +} + +// WithContext is a functional option to pass through a context.Context. +// +// By default, context.Background() is used. +func WithContext(ctx context.Context) Option { + return func(o *options) { + o.ctx = ctx } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go index f29dd73f..5d400156 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go @@ -15,47 +15,28 @@ package daemon import ( - "context" "fmt" "io" "io/ioutil" - "github.com/docker/docker/api/types" - "github.com/docker/docker/client" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/tarball" ) -// ImageLoader is an interface for testing. -type ImageLoader interface { - ImageLoad(context.Context, io.Reader, bool) (types.ImageLoadResponse, error) - ImageTag(context.Context, string, string) error -} - -// GetImageLoader is a variable so we can override in tests. -var GetImageLoader = func() (ImageLoader, error) { - cli, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - return nil, err - } - cli.NegotiateAPIVersion(context.Background()) - return cli, nil -} - // Tag adds a tag to an already existent image. -func Tag(src, dest name.Tag) error { - cli, err := GetImageLoader() +func Tag(src, dest name.Tag, options ...Option) error { + o, err := makeOptions(options...) if err != nil { return err } - return cli.ImageTag(context.Background(), src.String(), dest.String()) + return o.client.ImageTag(o.ctx, src.String(), dest.String()) } // Write saves the image into the daemon as the given tag. -func Write(tag name.Tag, img v1.Image) (string, error) { - cli, err := GetImageLoader() +func Write(tag name.Tag, img v1.Image, options ...Option) (string, error) { + o, err := makeOptions(options...) if err != nil { return "", err } @@ -66,14 +47,14 @@ func Write(tag name.Tag, img v1.Image) (string, error) { }() // write the image in docker save format first, then load it - resp, err := cli.ImageLoad(context.Background(), pr, false) + resp, err := o.client.ImageLoad(o.ctx, pr, false) if err != nil { return "", fmt.Errorf("error loading image: %v", err) } defer resp.Body.Close() - b, readErr := ioutil.ReadAll(resp.Body) + b, err := ioutil.ReadAll(resp.Body) response := string(b) - if readErr != nil { + if err != nil { return response, fmt.Errorf("error reading load response body: %v", err) } return response, nil diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/hash.go b/vendor/github.com/google/go-containerregistry/pkg/v1/hash.go index 7fdf22e9..e9630087 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/hash.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/hash.go @@ -87,7 +87,7 @@ func Hasher(name string) (hash.Hash, error) { func (h *Hash) parse(unquoted string) error { parts := strings.Split(unquoted, ":") if len(parts) != 2 { - return fmt.Errorf("too many parts in hash: %s", unquoted) + return fmt.Errorf("cannot parse hash: %q", unquoted) } rest := strings.TrimLeft(parts[1], "0123456789abcdef") diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go index 5569e51d..c322d775 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go @@ -3,40 +3,55 @@ package layout import v1 "github.com/google/go-containerregistry/pkg/v1" // Option is a functional option for Layout. -// -// TODO: We'll need to change this signature to support Sparse/Thin images. -// Or, alternatively, wrap it in a sparse.Image that returns an empty list for layers? -type Option func(*v1.Descriptor) error +type Option func(*options) + +type options struct { + descOpts []descriptorOption +} + +func makeOptions(opts ...Option) *options { + o := &options{ + descOpts: []descriptorOption{}, + } + for _, apply := range opts { + apply(o) + } + return o +} + +type descriptorOption func(*v1.Descriptor) // WithAnnotations adds annotations to the artifact descriptor. func WithAnnotations(annotations map[string]string) Option { - return func(desc *v1.Descriptor) error { - if desc.Annotations == nil { - desc.Annotations = make(map[string]string) - } - for k, v := range annotations { - desc.Annotations[k] = v - } - - return nil + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + for k, v := range annotations { + desc.Annotations[k] = v + } + }) } } // WithURLs adds urls to the artifact descriptor. func WithURLs(urls []string) Option { - return func(desc *v1.Descriptor) error { - if desc.URLs == nil { - desc.URLs = []string{} - } - desc.URLs = append(desc.URLs, urls...) - return nil + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + if desc.URLs == nil { + desc.URLs = []string{} + } + desc.URLs = append(desc.URLs, urls...) + }) } } // WithPlatform sets the platform of the artifact descriptor. func WithPlatform(platform v1.Platform) Option { - return func(desc *v1.Descriptor) error { - desc.Platform = &platform - return nil + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + desc.Platform = &platform + }) } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go index 9de456dd..a9ff91f7 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go @@ -62,10 +62,9 @@ func (l Path) AppendImage(img v1.Image, options ...Option) error { Digest: d, } - for _, opt := range options { - if err := opt(&desc); err != nil { - return err - } + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(&desc) } return l.AppendDescriptor(desc) @@ -99,10 +98,9 @@ func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error { Digest: d, } - for _, opt := range options { - if err := opt(&desc); err != nil { - return err - } + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(&desc) } return l.AppendDescriptor(desc) @@ -163,10 +161,9 @@ func (l Path) replaceDescriptor(append mutate.Appendable, matcher match.Matcher, return err } - for _, opt := range options { - if err := opt(desc); err != nil { - return err - } + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(desc) } add := mutate.IndexAddendum{ @@ -368,9 +365,9 @@ func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error { var blob io.ReadCloser // Workaround for #819. if wl, ok := ii.(withLayer); ok { - layer, err := wl.Layer(desc.Digest) - if err != nil { - return err + layer, lerr := wl.Layer(desc.Digest) + if lerr != nil { + return lerr } blob, err = layer.Compressed() } else if wb, ok := ii.(withBlob); ok { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go b/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go index 36c341df..51a46704 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go @@ -23,8 +23,8 @@ import ( // Manifest represents the OCI image manifest in a structured way. type Manifest struct { - SchemaVersion int64 `json:"schemaVersion,omitempty"` - MediaType types.MediaType `json:"mediaType"` + SchemaVersion int64 `json:"schemaVersion"` + MediaType types.MediaType `json:"mediaType,omitempty"` Config Descriptor `json:"config"` Layers []Descriptor `json:"layers"` Annotations map[string]string `json:"annotations,omitempty"` diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index 78b768fc..7336ed0a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -24,9 +24,9 @@ import ( "strings" "time" + "github.com/google/go-containerregistry/internal/gzip" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/internal/gzip" "github.com/google/go-containerregistry/pkg/v1/match" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/types" diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go index 0a229557..2e6e548a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go @@ -17,8 +17,8 @@ package partial import ( "io" + "github.com/google/go-containerregistry/internal/gzip" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/internal/gzip" "github.com/google/go-containerregistry/pkg/v1/types" ) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go index 69cea61c..df20d3aa 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go @@ -19,8 +19,8 @@ import ( "io" "sync" + "github.com/google/go-containerregistry/internal/gzip" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/internal/gzip" "github.com/google/go-containerregistry/pkg/v1/types" ) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go index 9c248cfe..2e0f9996 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -25,10 +25,10 @@ import ( "strconv" "strings" + "github.com/google/go-containerregistry/internal/verify" "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/internal/verify" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" @@ -324,14 +324,26 @@ func (f *fetcher) headManifest(ref name.Reference, acceptable []types.MediaType) return nil, err } - mediaType := types.MediaType(resp.Header.Get("Content-Type")) + mth := resp.Header.Get("Content-Type") + if mth == "" { + return nil, fmt.Errorf("HEAD %s: response did not include Content-Type header", u.String()) + } + mediaType := types.MediaType(mth) - size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + lh := resp.Header.Get("Content-Length") + if lh == "" { + return nil, fmt.Errorf("HEAD %s: response did not include Content-Length header", u.String()) + } + size, err := strconv.ParseInt(lh, 10, 64) if err != nil { return nil, err } - digest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest")) + dh := resp.Header.Get("Docker-Content-Digest") + if dh == "" { + return nil, fmt.Errorf("HEAD %s: response did not include Docker-Content-Digest header", u.String()) + } + digest, err := v1.NewHash(dh) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index e04961e9..64472364 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -21,10 +21,10 @@ import ( "net/url" "sync" - "github.com/google/go-containerregistry/pkg/internal/redact" + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/internal/verify" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/internal/verify" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/layer.go index 6a941a1a..89b0ede0 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/layer.go @@ -17,7 +17,7 @@ package remote import ( "io" - "github.com/google/go-containerregistry/pkg/internal/redact" + "github.com/google/go-containerregistry/internal/redact" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/multi_write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/multi_write.go index 6b81044f..45408fc4 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/multi_write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/multi_write.go @@ -33,7 +33,7 @@ import ( // Current limitations: // - All refs must share the same repository. // - Images cannot consist of stream.Layers. -func MultiWrite(m map[name.Reference]Taggable, options ...Option) error { +func MultiWrite(m map[name.Reference]Taggable, options ...Option) (rerr error) { // Determine the repository being pushed to; if asked to push to // multiple repositories, give up. var repo, zero name.Repository @@ -86,14 +86,54 @@ func MultiWrite(m map[name.Reference]Taggable, options ...Option) error { return err } w := writer{ - repo: repo, - client: &http.Client{Transport: tr}, - context: o.context, + repo: repo, + client: &http.Client{Transport: tr}, + context: o.context, + updates: o.updates, + lastUpdate: &v1.Update{}, + } + + // Collect the total size of blobs and manifests we're about to write. + if o.updates != nil { + defer close(o.updates) + defer func() { sendError(o.updates, rerr) }() + for _, b := range blobs { + size, err := b.Size() + if err != nil { + return err + } + w.lastUpdate.Total += size + } + countManifest := func(t Taggable) error { + b, err := t.RawManifest() + if err != nil { + return err + } + w.lastUpdate.Total += int64(len(b)) + return nil + } + for _, i := range images { + if err := countManifest(i); err != nil { + return err + } + } + for _, nm := range newManifests { + for _, i := range nm { + if err := countManifest(i); err != nil { + return err + } + } + } + for _, i := range indexes { + if err := countManifest(i); err != nil { + return err + } + } } // Upload individual blobs and collect any errors. blobChan := make(chan v1.Layer, 2*o.jobs) - var g errgroup.Group + g, ctx := errgroup.WithContext(o.context) for i := 0; i < o.jobs; i++ { // Start N workers consuming blobs to upload. g.Go(func() error { @@ -105,12 +145,17 @@ func MultiWrite(m map[name.Reference]Taggable, options ...Option) error { return nil }) } - go func() { + g.Go(func() error { + defer close(blobChan) for _, b := range blobs { - blobChan <- b + select { + case blobChan <- b: + case <-ctx.Done(): + return ctx.Err() + } } - close(blobChan) - }() + return nil + }) if err := g.Wait(); err != nil { return err } @@ -155,8 +200,8 @@ func MultiWrite(m map[name.Reference]Taggable, options ...Option) error { } // Push originally requested index manifests, which might depend on // newly discovered manifests. - return commitMany(indexes) + return commitMany(indexes) } // addIndexBlobs adds blobs to the set of blobs we intend to upload, and diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 04ff5082..7edebdf7 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -37,6 +37,7 @@ type options struct { jobs int userAgent string allowNondistributableArtifacts bool + updates chan<- v1.Update } var defaultPlatform = v1.Platform{ @@ -66,9 +67,6 @@ func makeOptions(target authn.Resource, opts ...Option) (*options, error) { if err != nil { return nil, err } - if auth == authn.Anonymous { - logs.Warn.Printf("No matching credentials were found for %q, falling back on anonymous", target) - } o.auth = auth } @@ -184,3 +182,14 @@ func WithNondistributable(o *options) error { o.allowNondistributableArtifacts = true return nil } + +// WithProgress takes a channel that will receive progress updates as bytes are written. +// +// Sending updates to an unbuffered channel will block writes, so callers +// should provide a buffered channel to avoid potential deadlocks. +func WithProgress(updates chan<- v1.Update) Option { + return func(o *options) error { + o.updates = updates + return nil + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index cd05509e..49941bd8 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -25,8 +25,9 @@ import ( "strings" authchallenge "github.com/docker/distribution/registry/client/auth/challenge" + "github.com/google/go-containerregistry/internal/redact" "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/internal/redact" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" ) @@ -262,6 +263,7 @@ func (bt *bearerTransport) refreshOauth(ctx context.Context) ([]byte, error) { defer resp.Body.Close() if err := CheckError(resp, http.StatusOK); err != nil { + logs.Warn.Printf("No matching credentials were found for %q", bt.registry) return nil, err } @@ -301,6 +303,7 @@ func (bt *bearerTransport) refreshBasic(ctx context.Context) ([]byte, error) { defer resp.Body.Close() if err := CheckError(resp, http.StatusOK); err != nil { + logs.Warn.Printf("No matching credentials were found for %q", bt.registry) return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 56708038..bb59d22e 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -100,7 +100,7 @@ func (e *Error) Temporary() bool { return true } -// TODO(jonjohnsonjr): Consider moving to pkg/internal/redact. +// TODO(jonjohnsonjr): Consider moving to internal/redact. func redactURL(original *url.URL) *url.URL { qs := original.Query() for k, v := range qs { @@ -163,6 +163,7 @@ var temporaryErrorCodes = map[ErrorCode]struct{}{ } var temporaryStatusCodes = map[int]struct{}{ + http.StatusRequestTimeout: {}, http.StatusInternalServerError: {}, http.StatusBadGateway: {}, http.StatusServiceUnavailable: {}, diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/logger.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/logger.go index 3b024866..c341f844 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/logger.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/logger.go @@ -20,7 +20,7 @@ import ( "net/http/httputil" "time" - "github.com/google/go-containerregistry/pkg/internal/redact" + "github.com/google/go-containerregistry/internal/redact" "github.com/google/go-containerregistry/pkg/logs" ) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go index 672622b6..7f7d1e45 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go @@ -18,7 +18,7 @@ import ( "net/http" "time" - "github.com/google/go-containerregistry/pkg/internal/retry" + "github.com/google/go-containerregistry/internal/retry" ) // Sleep for 0.1, 0.3, 0.9, 2.7 seconds. This should cover networking blips. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 261e9ec4..219afe51 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -23,15 +23,17 @@ import ( "net/http" "net/url" "strings" + "sync/atomic" "time" - "github.com/google/go-containerregistry/pkg/internal/redact" - "github.com/google/go-containerregistry/pkg/internal/retry" + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/internal/retry" "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "golang.org/x/sync/errgroup" ) @@ -42,61 +44,97 @@ type Taggable interface { } // Write pushes the provided img to the specified image reference. -func Write(ref name.Reference, img v1.Image, options ...Option) error { - ls, err := img.Layers() - if err != nil { - return err - } - +func Write(ref name.Reference, img v1.Image, options ...Option) (rerr error) { o, err := makeOptions(ref.Context(), options...) if err != nil { return err } + var lastUpdate *v1.Update + if o.updates != nil { + lastUpdate = &v1.Update{} + lastUpdate.Total, err = countImage(img, o.allowNondistributableArtifacts) + if err != nil { + return err + } + defer close(o.updates) + defer func() { sendError(o.updates, rerr) }() + } + return writeImage(ref, img, o, lastUpdate) +} + +func writeImage(ref name.Reference, img v1.Image, o *options, lastUpdate *v1.Update) error { + ls, err := img.Layers() + if err != nil { + return err + } scopes := scopesForUploadingImage(ref.Context(), ls) tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } w := writer{ - repo: ref.Context(), - client: &http.Client{Transport: tr}, - context: o.context, + repo: ref.Context(), + client: &http.Client{Transport: tr}, + context: o.context, + updates: o.updates, + lastUpdate: lastUpdate, + } + + // Upload individual blobs and collect any errors. + blobChan := make(chan v1.Layer, 2*o.jobs) + g, ctx := errgroup.WithContext(o.context) + for i := 0; i < o.jobs; i++ { + // Start N workers consuming blobs to upload. + g.Go(func() error { + for b := range blobChan { + if err := w.uploadOne(b); err != nil { + return err + } + } + return nil + }) } // Upload individual layers in goroutines and collect any errors. // If we can dedupe by the layer digest, try to do so. If we can't determine // the digest for whatever reason, we can't dedupe and might re-upload. - var g errgroup.Group - uploaded := map[v1.Hash]bool{} - for _, l := range ls { - l := l + g.Go(func() error { + defer close(blobChan) + uploaded := map[v1.Hash]bool{} + for _, l := range ls { + l := l - // Handle foreign layers. - mt, err := l.MediaType() - if err != nil { - return err - } - if !mt.IsDistributable() && !o.allowNondistributableArtifacts { - continue - } - - // Streaming layers calculate their digests while uploading them. Assume - // an error here indicates we need to upload the layer. - h, err := l.Digest() - if err == nil { - // If we can determine the layer's digest ahead of - // time, use it to dedupe uploads. - if uploaded[h] { - continue // Already uploading. + // Handle foreign layers. + mt, err := l.MediaType() + if err != nil { + return err + } + if !mt.IsDistributable() && !o.allowNondistributableArtifacts { + continue } - uploaded[h] = true - } - // TODO(#803): Pipe through remote.WithJobs and upload these in parallel. - g.Go(func() error { - return w.uploadOne(l) - }) + // Streaming layers calculate their digests while uploading them. Assume + // an error here indicates we need to upload the layer. + h, err := l.Digest() + if err == nil { + // If we can determine the layer's digest ahead of + // time, use it to dedupe uploads. + if uploaded[h] { + continue // Already uploading. + } + uploaded[h] = true + } + select { + case blobChan <- l: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) + if err := g.Wait(); err != nil { + return err } if l, err := partial.ConfigLayer(img); err != nil { @@ -137,6 +175,16 @@ type writer struct { repo name.Repository client *http.Client context context.Context + + updates chan<- v1.Update + lastUpdate *v1.Update +} + +func sendError(ch chan<- v1.Update, err error) error { + if err != nil && ch != nil { + ch <- v1.Update{Error: err} + } + return err } // url returns a url.Url for the specified path in the context of this remote image reference. @@ -259,10 +307,49 @@ func (w *writer) initiateUpload(from, mount string) (location string, mounted bo } } +type progressReader struct { + rc io.ReadCloser + + count *int64 // number of bytes this reader has read, to support resetting on retry. + updates chan<- v1.Update + lastUpdate *v1.Update +} + +func (r *progressReader) Read(b []byte) (int, error) { + n, err := r.rc.Read(b) + if err != nil { + return n, err + } + atomic.AddInt64(r.count, int64(n)) + // TODO: warn/debug log if sending takes too long, or if sending is blocked while context is cancelled. + r.updates <- v1.Update{ + Total: r.lastUpdate.Total, + Complete: atomic.AddInt64(&r.lastUpdate.Complete, int64(n)), + } + return n, nil +} + +func (r *progressReader) Close() error { return r.rc.Close() } + // streamBlob streams the contents of the blob to the specified location. // On failure, this will return an error. On success, this will return the location // header indicating how to commit the streamed blob. -func (w *writer) streamBlob(ctx context.Context, blob io.ReadCloser, streamLocation string) (commitLocation string, err error) { +func (w *writer) streamBlob(ctx context.Context, blob io.ReadCloser, streamLocation string) (commitLocation string, rerr error) { + reset := func() {} + defer func() { + if rerr != nil { + reset() + } + }() + if w.updates != nil { + var count int64 + blob = &progressReader{rc: blob, updates: w.updates, lastUpdate: w.lastUpdate, count: &count} + reset = func() { + atomic.AddInt64(&w.lastUpdate.Complete, -count) + w.updates <- *w.lastUpdate + } + } + req, err := http.NewRequest(http.MethodPatch, streamLocation, blob) if err != nil { return "", err @@ -308,6 +395,17 @@ func (w *writer) commitBlob(location, digest string) error { return transport.CheckError(resp, http.StatusCreated) } +// incrProgress increments and sends a progress update, if WithProgress is used. +func (w *writer) incrProgress(written int64) { + if w.updates == nil { + return + } + w.updates <- v1.Update{ + Total: w.lastUpdate.Total, + Complete: atomic.AddInt64(&w.lastUpdate.Complete, int64(written)), + } +} + // uploadOne performs a complete upload of a single layer. func (w *writer) uploadOne(l v1.Layer) error { var from, mount string @@ -319,6 +417,11 @@ func (w *writer) uploadOne(l v1.Layer) error { return err } if existing { + size, err := l.Size() + if err != nil { + return err + } + w.incrProgress(size) logs.Progress.Printf("existing blob: %v", h) return nil } @@ -338,6 +441,11 @@ func (w *writer) uploadOne(l v1.Layer) error { if err != nil { return err } else if mounted { + size, err := l.Size() + if err != nil { + return err + } + w.incrProgress(size) h, err := l.Digest() if err != nil { return err @@ -400,6 +508,11 @@ func (w *writer) writeIndex(ref name.Reference, ii v1.ImageIndex, options ...Opt return err } + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return err + } + // TODO(#803): Pipe through remote.WithJobs and upload these in parallel. for _, desc := range index.Manifests { ref := ref.Context().Digest(desc.Digest.String()) @@ -418,7 +531,6 @@ func (w *writer) writeIndex(ref name.Reference, ii v1.ImageIndex, options ...Opt if err != nil { return err } - if err := w.writeIndex(ref, ii); err != nil { return err } @@ -427,10 +539,7 @@ func (w *writer) writeIndex(ref name.Reference, ii v1.ImageIndex, options ...Opt if err != nil { return err } - // TODO: Ideally we could reuse this writer, but we need to know - // scopes before we do the token exchange. To be lazy here, just - // re-do the token exchange. MultiWrite fixes this. - if err := Write(ref, img, options...); err != nil { + if err := writeImage(ref, img, o, w.lastUpdate); err != nil { return err } default: @@ -522,6 +631,7 @@ func (w *writer) commitManifest(t Taggable, ref name.Reference) error { // The image was successfully pushed! logs.Progress.Printf("%v: digest: %v size: %d", ref, desc.Digest, desc.Size) + w.incrProgress(int64(len(raw))) return nil } @@ -553,11 +663,12 @@ func scopesForUploadingImage(repo name.Repository, layers []v1.Layer) []string { // WriteIndex pushes the provided ImageIndex to the specified image reference. // WriteIndex will attempt to push all of the referenced manifests before // attempting to push the ImageIndex, to retain referential integrity. -func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { +func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) (rerr error) { o, err := makeOptions(ref.Context(), options...) if err != nil { return err } + scopes := []string{ref.Scope(transport.PushScope)} tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { @@ -567,12 +678,132 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { repo: ref.Context(), client: &http.Client{Transport: tr}, context: o.context, + updates: o.updates, } + + if o.updates != nil { + w.lastUpdate = &v1.Update{} + w.lastUpdate.Total, err = countIndex(ii, o.allowNondistributableArtifacts) + if err != nil { + return err + } + defer close(o.updates) + defer func() { sendError(o.updates, rerr) }() + } + return w.writeIndex(ref, ii, options...) } +// countImage counts the total size of all layers + config blob + manifest for +// an image. It de-dupes duplicate layers. +func countImage(img v1.Image, allowNondistributableArtifacts bool) (int64, error) { + var total int64 + ls, err := img.Layers() + if err != nil { + return 0, err + } + seen := map[v1.Hash]bool{} + for _, l := range ls { + // Handle foreign layers. + mt, err := l.MediaType() + if err != nil { + return 0, err + } + if !mt.IsDistributable() && !allowNondistributableArtifacts { + continue + } + + // TODO: support streaming layers which update the total count as they write. + if _, ok := l.(*stream.Layer); ok { + return 0, errors.New("cannot use stream.Layer and WithProgress") + } + + // Dedupe layers. + d, err := l.Digest() + if err != nil { + return 0, err + } + if seen[d] { + continue + } + seen[d] = true + + size, err := l.Size() + if err != nil { + return 0, err + } + total += size + } + b, err := img.RawConfigFile() + if err != nil { + return 0, err + } + total += int64(len(b)) + size, err := img.Size() + if err != nil { + return 0, err + } + total += size + return total, nil +} + +// countIndex counts the total size of all images + sub-indexes for an index. +// It does not attempt to de-dupe duplicate images, etc. +func countIndex(idx v1.ImageIndex, allowNondistributableArtifacts bool) (int64, error) { + var total int64 + mf, err := idx.IndexManifest() + if err != nil { + return 0, err + } + + for _, desc := range mf.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + sidx, err := idx.ImageIndex(desc.Digest) + if err != nil { + return 0, err + } + size, err := countIndex(sidx, allowNondistributableArtifacts) + if err != nil { + return 0, err + } + total += size + case types.OCIManifestSchema1, types.DockerManifestSchema2: + simg, err := idx.Image(desc.Digest) + if err != nil { + return 0, err + } + size, err := countImage(simg, allowNondistributableArtifacts) + if err != nil { + return 0, err + } + total += size + default: + // Workaround for #819. + if wl, ok := idx.(withLayer); ok { + layer, err := wl.Layer(desc.Digest) + if err != nil { + return 0, err + } + size, err := layer.Size() + if err != nil { + return 0, err + } + total += size + } + } + } + + size, err := idx.Size() + if err != nil { + return 0, err + } + total += size + return total, nil +} + // WriteLayer uploads the provided Layer to the specified repo. -func WriteLayer(repo name.Repository, layer v1.Layer, options ...Option) error { +func WriteLayer(repo name.Repository, layer v1.Layer, options ...Option) (rerr error) { o, err := makeOptions(repo, options...) if err != nil { return err @@ -586,8 +817,23 @@ func WriteLayer(repo name.Repository, layer v1.Layer, options ...Option) error { repo: repo, client: &http.Client{Transport: tr}, context: o.context, + updates: o.updates, } + if o.updates != nil { + defer close(o.updates) + defer func() { sendError(o.updates, rerr) }() + + // TODO: support streaming layers which update the total count as they write. + if _, ok := layer.(*stream.Layer); ok { + return errors.New("cannot use stream.Layer and WithProgress") + } + size, err := layer.Size() + if err != nil { + return err + } + w.lastUpdate = &v1.Update{Total: size} + } return w.uploadOne(layer) } @@ -603,11 +849,26 @@ func WriteLayer(repo name.Repository, layer v1.Layer, options ...Option) error { // should ensure that all blobs or manifests that are referenced by t exist // in the target registry. func Tag(tag name.Tag, t Taggable, options ...Option) error { - o, err := makeOptions(tag.Context(), options...) + return Put(tag, t, options...) +} + +// Put adds a manifest from the given Taggable via PUT /v1/.../manifest/ +// +// Notable implementations of Taggable are v1.Image, v1.ImageIndex, and +// remote.Descriptor. +// +// If t implements MediaType, we will use that for the Content-Type, otherwise +// we will default to types.DockerManifestSchema2. +// +// Put does not attempt to write anything other than the manifest, so callers +// should ensure that all blobs or manifests that are referenced by t exist +// in the target registry. +func Put(ref name.Reference, t Taggable, options ...Option) error { + o, err := makeOptions(ref.Context(), options...) if err != nil { return err } - scopes := []string{tag.Scope(transport.PushScope)} + scopes := []string{ref.Scope(transport.PushScope)} // TODO: This *always* does a token exchange. For some registries, // that's pretty slow. Some ideas; @@ -615,15 +876,15 @@ func Tag(tag name.Tag, t Taggable, options ...Option) error { // * Allow callers to pass in a transport.Transport, typecheck // it to allow them to reuse the transport across multiple calls. // * WithTag option to do multiple manifest PUTs in commitManifest. - tr, err := transport.NewWithContext(o.context, tag.Context().Registry, o.auth, o.transport, scopes) + tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } w := writer{ - repo: tag.Context(), + repo: ref.Context(), client: &http.Client{Transport: tr}, context: o.context, } - return w.commitManifest(t, tag) + return w.commitManifest(t, ref) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go index 08f7a812..980842d2 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -23,11 +23,13 @@ import ( "io" "io/ioutil" "os" + "path" + "path/filepath" "sync" + "github.com/google/go-containerregistry/internal/gzip" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/internal/gzip" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -216,6 +218,10 @@ func extractFileFromTar(opener Opener, filePath string) (io.ReadCloser, error) { return nil, err } if hdr.Name == filePath { + if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink { + currentDir := filepath.Dir(filePath) + return extractFileFromTar(opener, path.Join(currentDir, hdr.Linkname)) + } close = false return tarFile{ Reader: tf, diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index dc8684ce..5ec1d551 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -23,10 +23,10 @@ import ( "sync" "github.com/containerd/stargz-snapshotter/estargz" + "github.com/google/go-containerregistry/internal/and" + gestargz "github.com/google/go-containerregistry/internal/estargz" + ggzip "github.com/google/go-containerregistry/internal/gzip" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/internal/and" - gestargz "github.com/google/go-containerregistry/pkg/v1/internal/estargz" - ggzip "github.com/google/go-containerregistry/pkg/v1/internal/gzip" "github.com/google/go-containerregistry/pkg/v1/types" ) diff --git a/vendor/modules.txt b/vendor/modules.txt index 395e07ed..12c11a77 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -105,16 +105,20 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value -# github.com/google/go-containerregistry v0.4.1-0.20210216200643-d81088d9983e +# github.com/google/go-containerregistry v0.5.0 ## explicit github.com/google/go-containerregistry/cmd/crane/cmd +github.com/google/go-containerregistry/internal/and +github.com/google/go-containerregistry/internal/estargz +github.com/google/go-containerregistry/internal/gzip +github.com/google/go-containerregistry/internal/httptest +github.com/google/go-containerregistry/internal/legacy +github.com/google/go-containerregistry/internal/redact +github.com/google/go-containerregistry/internal/retry +github.com/google/go-containerregistry/internal/retry/wait +github.com/google/go-containerregistry/internal/verify github.com/google/go-containerregistry/pkg/authn github.com/google/go-containerregistry/pkg/crane -github.com/google/go-containerregistry/pkg/internal/httptest -github.com/google/go-containerregistry/pkg/internal/legacy -github.com/google/go-containerregistry/pkg/internal/redact -github.com/google/go-containerregistry/pkg/internal/retry -github.com/google/go-containerregistry/pkg/internal/retry/wait github.com/google/go-containerregistry/pkg/legacy github.com/google/go-containerregistry/pkg/legacy/tarball github.com/google/go-containerregistry/pkg/logs @@ -124,10 +128,6 @@ github.com/google/go-containerregistry/pkg/v1 github.com/google/go-containerregistry/pkg/v1/cache github.com/google/go-containerregistry/pkg/v1/daemon github.com/google/go-containerregistry/pkg/v1/empty -github.com/google/go-containerregistry/pkg/v1/internal/and -github.com/google/go-containerregistry/pkg/v1/internal/estargz -github.com/google/go-containerregistry/pkg/v1/internal/gzip -github.com/google/go-containerregistry/pkg/v1/internal/verify github.com/google/go-containerregistry/pkg/v1/layout github.com/google/go-containerregistry/pkg/v1/match github.com/google/go-containerregistry/pkg/v1/mutate