From ab8152ad0aaa4610e2ce79d1a9e485097f585cef Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Thu, 21 Mar 2019 18:51:39 -0400 Subject: [PATCH] Update ko to ggcr head, update ggcr vendor --- Gopkg.lock | 10 +- cmd/ko/commands.go | 16 +- cmd/ko/flatname.go | 10 + cmd/ko/publish.go | 14 +- cmd/ko/resolve.go | 14 +- pkg/build/gobuild.go | 55 ++- pkg/build/gobuild_test.go | 11 +- pkg/resolve/fixed_test.go | 5 +- pkg/resolve/resolve.go | 7 +- pkg/resolve/resolve_test.go | 5 +- .../pkg/ko/build/build.go | 31 ++ .../go-containerregistry/pkg/ko/build/doc.go | 17 + .../pkg/ko/build/future.go | 75 ++++ .../pkg/ko/build/gobuild.go | 346 ++++++++++++++++++ .../pkg/ko/build/options.go | 46 +++ .../pkg/ko/build/recorder.go | 46 +++ .../pkg/ko/build/shared.go | 79 ++++ .../pkg/ko/publish/daemon.go | 80 ++++ .../pkg/ko/publish/default.go | 121 ++++++ .../pkg/ko/publish/doc.go | 17 + .../pkg/ko/publish/future.go | 75 ++++ .../pkg/ko/publish/options.go | 82 +++++ .../pkg/ko/publish/publish.go | 28 ++ .../pkg/ko/publish/shared.go | 76 ++++ .../pkg/ko/resolve/doc.go | 16 + .../pkg/ko/resolve/resolve.go | 160 ++++++++ .../pkg/v1/remote/check.go | 56 +++ .../pkg/v1/remote/descriptor.go | 271 ++++++++++++++ .../pkg/v1/remote/image.go | 186 +--------- .../pkg/v1/remote/index.go | 126 +++++-- .../pkg/v1/remote/options.go | 6 + .../pkg/v1/tarball/layer.go | 13 + .../pkg/v1/tarball/write.go | 51 ++- 33 files changed, 1884 insertions(+), 267 deletions(-) create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/build/build.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/build/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/build/future.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/build/gobuild.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/build/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/build/recorder.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/build/shared.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/publish/daemon.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/publish/default.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/publish/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/publish/future.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/publish/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/publish/publish.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/publish/shared.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/resolve/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/ko/resolve/resolve.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go diff --git a/Gopkg.lock b/Gopkg.lock index 6cbca7c1..3a174ae6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -142,10 +142,13 @@ [[projects]] branch = "master" - digest = "1:123ca2e74131111f6302f6e0eb27bbae6d8989b7dae00ca7a624793b4549353b" + digest = "1:9d25404d202ff8f0097f3086b18645b6ac1e6634ab35a193328c5b888776252b" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", + "pkg/ko/build", + "pkg/ko/publish", + "pkg/ko/resolve", "pkg/name", "pkg/v1", "pkg/v1/daemon", @@ -161,7 +164,7 @@ "pkg/v1/v1util", ] pruneopts = "UT" - revision = "6225ca1a4de721ff14f6c4cbbfd141ab462bdb22" + revision = "8d4083db9aa0d2fae6588c1acdbe6a1f5db461e3" [[projects]] branch = "master" @@ -608,6 +611,9 @@ "github.com/google/go-cmp/cmp", "github.com/google/go-cmp/cmp/cmpopts", "github.com/google/go-containerregistry/pkg/authn", + "github.com/google/go-containerregistry/pkg/ko/build", + "github.com/google/go-containerregistry/pkg/ko/publish", + "github.com/google/go-containerregistry/pkg/ko/resolve", "github.com/google/go-containerregistry/pkg/name", "github.com/google/go-containerregistry/pkg/v1", "github.com/google/go-containerregistry/pkg/v1/daemon", diff --git a/cmd/ko/commands.go b/cmd/ko/commands.go index 9a20e4ea..c04a1dc8 100644 --- a/cmd/ko/commands.go +++ b/cmd/ko/commands.go @@ -131,7 +131,21 @@ func addKubeCommands(topLevel *cobra.Command) { if err != nil { log.Fatalf("error piping to 'kubectl apply': %v", err) } - go resolveFilesToWriter(fo, no, lo, ta, stdin) + + go func() { + // kubectl buffers data before starting to apply it, which + // can lead to resources being created more slowly than desired. + // In the case of --watch, it can lead to resources not being + // applied at all until enough iteration has occurred. To work + // around this, we prime the stream with a bunch of empty objects + // which kubectl will discard. + // See https://github.com/google/go-containerregistry/pull/348 + for i := 0; i < 1000; i++ { + stdin.Write([]byte("---\n")) + } + // Once primed kick things off. + resolveFilesToWriter(fo, no, lo, ta, stdin) + }() // Run it. if err := kubectlCmd.Run(); err != nil { diff --git a/cmd/ko/flatname.go b/cmd/ko/flatname.go index a4521526..04112403 100644 --- a/cmd/ko/flatname.go +++ b/cmd/ko/flatname.go @@ -19,6 +19,7 @@ import ( "encoding/hex" "path/filepath" + "github.com/google/go-containerregistry/pkg/ko/publish" "github.com/spf13/cobra" ) @@ -50,3 +51,12 @@ func preserveImportPath(importpath string) string { func baseImportPaths(importpath string) string { return filepath.Base(importpath) } + +func makeNamer(no *NameOptions) publish.Namer { + if no.PreserveImportPaths { + return preserveImportPath + } else if no.BaseImportPaths { + return baseImportPaths + } + return packageWithMD5 +} diff --git a/cmd/ko/publish.go b/cmd/ko/publish.go index 950aa4d9..d22907c6 100644 --- a/cmd/ko/publish.go +++ b/cmd/ko/publish.go @@ -23,10 +23,9 @@ import ( "strings" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/ko/build" + "github.com/google/go-containerregistry/pkg/ko/publish" "github.com/google/go-containerregistry/pkg/name" - - "github.com/google/ko/pkg/build" - "github.com/google/ko/pkg/publish" ) func qualifyLocalImport(importpath, gopathsrc, pwd string) (string, error) { @@ -75,14 +74,7 @@ func publishImages(importpaths []string, no *NameOptions, lo *LocalOptions, ta * var pub publish.Interface repoName := os.Getenv("KO_DOCKER_REPO") - var namer publish.Namer - if no.PreserveImportPaths { - namer = preserveImportPath - } else if no.BaseImportPaths { - namer = baseImportPaths - } else { - namer = packageWithMD5 - } + namer := makeNamer(no) if lo.Local || repoName == publish.LocalDomain { pub = publish.NewDaemon(namer, ta.Tags) diff --git a/cmd/ko/resolve.go b/cmd/ko/resolve.go index 2c6c77b0..85f706f8 100644 --- a/cmd/ko/resolve.go +++ b/cmd/ko/resolve.go @@ -23,12 +23,11 @@ import ( "sync" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/ko/build" + "github.com/google/go-containerregistry/pkg/ko/publish" + "github.com/google/go-containerregistry/pkg/ko/resolve" "github.com/google/go-containerregistry/pkg/name" "github.com/mattmoor/dep-notify/pkg/graph" - - "github.com/google/ko/pkg/build" - "github.com/google/ko/pkg/publish" - "github.com/google/ko/pkg/resolve" ) func gobuildOptions() ([]build.Option, error) { @@ -78,12 +77,7 @@ func makePublisher(no *NameOptions, lo *LocalOptions, ta *TagsOptions) (publish. // Create the publish.Interface that we will use to publish image references // to either a docker daemon or a container image registry. innerPublisher, err := func() (publish.Interface, error) { - namer := func() publish.Namer { - if no.PreserveImportPaths { - return preserveImportPath - } - return packageWithMD5 - }() + namer := makeNamer(no) repoName := os.Getenv("KO_DOCKER_REPO") if lo.Local || repoName == publish.LocalDomain { diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 6a378ceb..7300f5e2 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -31,7 +31,10 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" ) -const appPath = "/ko-app" +const ( + appDir = "/ko-app" + defaultAppFilename = "ko-app" +) // GetBase takes an importpath and returns a base v1.Image. type GetBase func(string) (v1.Image, error) @@ -117,11 +120,53 @@ func build(ip string) (string, error) { return file, nil } -func tarBinary(binary string) (*bytes.Buffer, error) { +func appFilename(importpath string) string { + base := filepath.Base(importpath) + + // If we fail to determine a good name from the importpath then use a + // safe default. + if base == "." || base == string(filepath.Separator) { + return defaultAppFilename + } + + return base +} + +func tarAddDirectories(tw *tar.Writer, dir string) error { + if dir == "." || dir == string(filepath.Separator) { + return nil + } + + // Write parent directories first + if err := tarAddDirectories(tw, filepath.Dir(dir)); err != nil { + return err + } + + // write the directory header to the tarball archive + if err := tw.WriteHeader(&tar.Header{ + Name: dir, + Typeflag: tar.TypeDir, + // Use a fixed Mode, so that this isn't sensitive to the directory and umask + // under which it was created. Additionally, windows can only set 0222, + // 0444, or 0666, none of which are executable. + Mode: 0555, + }); err != nil { + return err + } + + return nil +} + +func tarBinary(name, binary string) (*bytes.Buffer, error) { buf := bytes.NewBuffer(nil) tw := tar.NewWriter(buf) defer tw.Close() + // write the parent directories to the tarball archive + if err := tarAddDirectories(tw, filepath.Dir(name)); err != nil { + return nil, err + } + file, err := os.Open(binary) if err != nil { return nil, err @@ -132,7 +177,7 @@ func tarBinary(binary string) (*bytes.Buffer, error) { return nil, err } header := &tar.Header{ - Name: appPath, + Name: name, Size: stat.Size(), Typeflag: tar.TypeReg, // Use a fixed Mode, so that this isn't sensitive to the directory and umask @@ -249,8 +294,10 @@ func (gb *gobuild) Build(s string) (v1.Image, error) { } layers = append(layers, dataLayer) + appPath := filepath.Join(appDir, appFilename(s)) + // Construct a tarball with the binary and produce a layer. - binaryLayerBuf, err := tarBinary(file) + binaryLayerBuf, err := tarBinary(appPath, file) if err != nil { return nil, err } diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index dc5a12dd..9b234c90 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -19,9 +19,8 @@ import ( "io" "io/ioutil" "path/filepath" - "time" - "testing" + "time" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/random" @@ -117,7 +116,7 @@ func TestGoBuildNoKoData(t *testing.T) { t.Run("check determinism", func(t *testing.T) { expectedHash := v1.Hash{ Algorithm: "sha256", - Hex: "1d4fb5a6e81840aa5996d6efad00cca54b14412917ed42acf51d88d3f9482fd0", + Hex: "fb82c95fc73eaf26d0b18b1bc2d23ee32059e46806a83a313e738aac4d039492", } appLayer := ls[baseLayers+1] @@ -139,7 +138,7 @@ func TestGoBuildNoKoData(t *testing.T) { t.Errorf("len(entrypoint) = %v, want %v", got, want) } - if got, want := entrypoint[0], appPath; got != want { + if got, want := entrypoint[0], "/ko-app/ko"; got != want { t.Errorf("entrypoint = %v, want %v", got, want) } }) @@ -194,7 +193,7 @@ func TestGoBuild(t *testing.T) { t.Run("check determinism", func(t *testing.T) { expectedHash := v1.Hash{ Algorithm: "sha256", - Hex: "481f1025f9a594d8742cadb1928d1d601115a14a77001958dc539cee04fddfcf", + Hex: "4c7f97dda30576670c3a8967424f7dea023030bb3df74fc4bd10329bcb266fc2", } appLayer := ls[baseLayers+1] @@ -275,7 +274,7 @@ func TestGoBuild(t *testing.T) { t.Errorf("len(entrypoint) = %v, want %v", got, want) } - if got, want := entrypoint[0], appPath; got != want { + if got, want := entrypoint[0], "/ko-app/test"; got != want { t.Errorf("entrypoint = %v, want %v", got, want) } }) diff --git a/pkg/resolve/fixed_test.go b/pkg/resolve/fixed_test.go index d766906d..6aee1e1b 100644 --- a/pkg/resolve/fixed_test.go +++ b/pkg/resolve/fixed_test.go @@ -18,12 +18,11 @@ import ( "fmt" "testing" + "github.com/google/go-containerregistry/pkg/ko/build" + "github.com/google/go-containerregistry/pkg/ko/publish" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/random" - - "github.com/google/ko/pkg/build" - "github.com/google/ko/pkg/publish" ) var ( diff --git a/pkg/resolve/resolve.go b/pkg/resolve/resolve.go index bee4538c..6a3ed56e 100644 --- a/pkg/resolve/resolve.go +++ b/pkg/resolve/resolve.go @@ -20,11 +20,12 @@ import ( "io" "sync" - "golang.org/x/sync/errgroup" yaml "gopkg.in/yaml.v2" - "github.com/google/ko/pkg/build" - "github.com/google/ko/pkg/publish" + "golang.org/x/sync/errgroup" + + "github.com/google/go-containerregistry/pkg/ko/build" + "github.com/google/go-containerregistry/pkg/ko/publish" ) // ImageReferences resolves supported references to images within the input yaml diff --git a/pkg/resolve/resolve_test.go b/pkg/resolve/resolve_test.go index c4b1942c..e8a95738 100644 --- a/pkg/resolve/resolve_test.go +++ b/pkg/resolve/resolve_test.go @@ -19,13 +19,12 @@ import ( "io" "testing" - yaml "gopkg.in/yaml.v2" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/random" + yaml "gopkg.in/yaml.v2" ) var ( @@ -309,7 +308,7 @@ func TestMultiDocumentYAMLs(t *testing.T) { } func mustRandom() v1.Image { - img, err := random.Image(5, 1024) + img, err := random.Image(1024, 5) if err != nil { panic(err) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/build/build.go b/vendor/github.com/google/go-containerregistry/pkg/ko/build/build.go new file mode 100644 index 00000000..06c33c6c --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/build/build.go @@ -0,0 +1,31 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Interface abstracts different methods for turning a supported importpath +// reference into a v1.Image. +type Interface interface { + // IsSupportedReference determines whether the given reference is to an importpath reference + // that Ko supports building. + // TODO(mattmoor): Verify that some base repo: foo.io/bar can be suffixed with this reference and parsed. + IsSupportedReference(string) bool + + // Build turns the given importpath reference into a v1.Image containing the Go binary. + Build(string) (v1.Image, error) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/build/doc.go b/vendor/github.com/google/go-containerregistry/pkg/ko/build/doc.go new file mode 100644 index 00000000..a8cb7fbe --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/build/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package build defines methods for building a v1.Image reference from a +// Go binary reference. +package build diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/build/future.go b/vendor/github.com/google/go-containerregistry/pkg/ko/build/future.go new file mode 100644 index 00000000..8f66fbbc --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/build/future.go @@ -0,0 +1,75 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +func newFuture(work func() (v1.Image, error)) *future { + // Create a channel on which to send the result. + ch := make(chan *result) + // Initiate the actual work, sending its result + // along the above channel. + go func() { + img, err := work() + ch <- &result{img: img, err: err} + }() + // Return a future for the above work. Callers should + // call .Get() on this result (as many times as needed). + // One of these calls will receive the result, store it, + // and close the channel so that the rest of the callers + // can consume it. + return &future{ + promise: ch, + } +} + +type result struct { + img v1.Image + err error +} + +type future struct { + m sync.RWMutex + + result *result + promise chan *result +} + +// Get blocks on the result of the future. +func (f *future) Get() (v1.Image, error) { + // Block on the promise of a result until we get one. + result, ok := <-f.promise + if ok { + func() { + f.m.Lock() + defer f.m.Unlock() + // If we got the result, then store it so that + // others may access it. + f.result = result + // Close the promise channel so that others + // are signaled that the result is available. + close(f.promise) + }() + } + + f.m.RLock() + defer f.m.RUnlock() + + return f.result.img, f.result.err +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/build/gobuild.go b/vendor/github.com/google/go-containerregistry/pkg/ko/build/gobuild.go new file mode 100644 index 00000000..7300f5e2 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/build/gobuild.go @@ -0,0 +1,346 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "archive/tar" + "bytes" + "errors" + gb "go/build" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +const ( + appDir = "/ko-app" + defaultAppFilename = "ko-app" +) + +// GetBase takes an importpath and returns a base v1.Image. +type GetBase func(string) (v1.Image, error) +type builder func(string) (string, error) + +type gobuild struct { + getBase GetBase + creationTime v1.Time + build builder +} + +// Option is a functional option for NewGo. +type Option func(*gobuildOpener) error + +type gobuildOpener struct { + getBase GetBase + creationTime v1.Time + build builder +} + +func (gbo *gobuildOpener) Open() (Interface, error) { + if gbo.getBase == nil { + return nil, errors.New("a way of providing base images must be specified, see build.WithBaseImages") + } + return &gobuild{ + getBase: gbo.getBase, + creationTime: gbo.creationTime, + build: gbo.build, + }, nil +} + +// NewGo returns a build.Interface implementation that: +// 1. builds go binaries named by importpath, +// 2. containerizes the binary on a suitable base, +func NewGo(options ...Option) (Interface, error) { + gbo := &gobuildOpener{ + build: build, + } + + for _, option := range options { + if err := option(gbo); err != nil { + return nil, err + } + } + return gbo.Open() +} + +// IsSupportedReference implements build.Interface +// +// Only valid importpaths that provide commands (i.e., are "package main") are +// supported. +func (*gobuild) IsSupportedReference(s string) bool { + p, err := gb.Import(s, gb.Default.GOPATH, gb.ImportComment) + if err != nil { + return false + } + return p.IsCommand() +} + +func build(ip string) (string, error) { + tmpDir, err := ioutil.TempDir("", "ko") + if err != nil { + return "", err + } + file := filepath.Join(tmpDir, "out") + + cmd := exec.Command("go", "build", "-o", file, ip) + + // Last one wins + // TODO(mattmoor): GOARCH=amd64 + cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux") + + var output bytes.Buffer + cmd.Stderr = &output + cmd.Stdout = &output + + log.Printf("Building %s", ip) + if err := cmd.Run(); err != nil { + os.RemoveAll(tmpDir) + log.Printf("Unexpected error running \"go build\": %v\n%v", err, output.String()) + return "", err + } + return file, nil +} + +func appFilename(importpath string) string { + base := filepath.Base(importpath) + + // If we fail to determine a good name from the importpath then use a + // safe default. + if base == "." || base == string(filepath.Separator) { + return defaultAppFilename + } + + return base +} + +func tarAddDirectories(tw *tar.Writer, dir string) error { + if dir == "." || dir == string(filepath.Separator) { + return nil + } + + // Write parent directories first + if err := tarAddDirectories(tw, filepath.Dir(dir)); err != nil { + return err + } + + // write the directory header to the tarball archive + if err := tw.WriteHeader(&tar.Header{ + Name: dir, + Typeflag: tar.TypeDir, + // Use a fixed Mode, so that this isn't sensitive to the directory and umask + // under which it was created. Additionally, windows can only set 0222, + // 0444, or 0666, none of which are executable. + Mode: 0555, + }); err != nil { + return err + } + + return nil +} + +func tarBinary(name, binary string) (*bytes.Buffer, error) { + buf := bytes.NewBuffer(nil) + tw := tar.NewWriter(buf) + defer tw.Close() + + // write the parent directories to the tarball archive + if err := tarAddDirectories(tw, filepath.Dir(name)); err != nil { + return nil, err + } + + file, err := os.Open(binary) + if err != nil { + return nil, err + } + defer file.Close() + stat, err := file.Stat() + if err != nil { + return nil, err + } + header := &tar.Header{ + Name: name, + Size: stat.Size(), + Typeflag: tar.TypeReg, + // Use a fixed Mode, so that this isn't sensitive to the directory and umask + // under which it was created. Additionally, windows can only set 0222, + // 0444, or 0666, none of which are executable. + Mode: 0555, + } + // write the header to the tarball archive + if err := tw.WriteHeader(header); err != nil { + return nil, err + } + // copy the file data to the tarball + if _, err := io.Copy(tw, file); err != nil { + return nil, err + } + + return buf, nil +} + +func kodataPath(s string) (string, error) { + p, err := gb.Import(s, gb.Default.GOPATH, gb.ImportComment) + if err != nil { + return "", err + } + return filepath.Join(p.Dir, "kodata"), nil +} + +// Where kodata lives in the image. +const kodataRoot = "/var/run/ko" + +func tarKoData(importpath string) (*bytes.Buffer, error) { + buf := bytes.NewBuffer(nil) + tw := tar.NewWriter(buf) + defer tw.Close() + + root, err := kodataPath(importpath) + if err != nil { + return nil, err + } + + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if path == root { + // Add an entry for /var/run/ko + return tw.WriteHeader(&tar.Header{ + Name: kodataRoot, + Typeflag: tar.TypeDir, + }) + } + if err != nil { + return err + } + // Skip other directories. + if info.Mode().IsDir() { + return nil + } + + // Chase symlinks. + info, err = os.Stat(path) + if err != nil { + return err + } + + // Open the file to copy it into the tarball. + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + // Copy the file into the image tarball. + newPath := filepath.Join(kodataRoot, path[len(root):]) + if err := tw.WriteHeader(&tar.Header{ + Name: newPath, + Size: info.Size(), + Typeflag: tar.TypeReg, + // Use a fixed Mode, so that this isn't sensitive to the directory and umask + // under which it was created. Additionally, windows can only set 0222, + // 0444, or 0666, none of which are executable. + Mode: 0555, + }); err != nil { + return err + } + _, err = io.Copy(tw, file) + return err + }) + if err != nil { + return nil, err + } + + return buf, nil +} + +// Build implements build.Interface +func (gb *gobuild) Build(s string) (v1.Image, error) { + // Do the build into a temporary file. + file, err := gb.build(s) + if err != nil { + return nil, err + } + defer os.RemoveAll(filepath.Dir(file)) + + var layers []v1.Layer + // Create a layer from the kodata directory under this import path. + dataLayerBuf, err := tarKoData(s) + if err != nil { + return nil, err + } + dataLayerBytes := dataLayerBuf.Bytes() + dataLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewBuffer(dataLayerBytes)), nil + }) + if err != nil { + return nil, err + } + layers = append(layers, dataLayer) + + appPath := filepath.Join(appDir, appFilename(s)) + + // Construct a tarball with the binary and produce a layer. + binaryLayerBuf, err := tarBinary(appPath, file) + if err != nil { + return nil, err + } + binaryLayerBytes := binaryLayerBuf.Bytes() + binaryLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewBuffer(binaryLayerBytes)), nil + }) + if err != nil { + return nil, err + } + layers = append(layers, binaryLayer) + + // Determine the appropriate base image for this import path. + base, err := gb.getBase(s) + if err != nil { + return nil, err + } + + // Augment the base image with our application layer. + withApp, err := mutate.AppendLayers(base, layers...) + if err != nil { + return nil, err + } + + // Start from a copy of the base image's config file, and set + // the entrypoint to our app. + cfg, err := withApp.ConfigFile() + if err != nil { + return nil, err + } + + cfg = cfg.DeepCopy() + cfg.Config.Entrypoint = []string{appPath} + cfg.Config.Env = append(cfg.Config.Env, "KO_DATA_PATH="+kodataRoot) + + image, err := mutate.Config(withApp, cfg.Config) + if err != nil { + return nil, err + } + + empty := v1.Time{} + if gb.creationTime != empty { + return mutate.CreatedAt(image, gb.creationTime) + } + return image, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/build/options.go b/vendor/github.com/google/go-containerregistry/pkg/ko/build/options.go new file mode 100644 index 00000000..df6eb40a --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/build/options.go @@ -0,0 +1,46 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// WithBaseImages is a functional option for overriding the base images +// that are used for different images. +func WithBaseImages(gb GetBase) Option { + return func(gbo *gobuildOpener) error { + gbo.getBase = gb + return nil + } +} + +// WithCreationTime is a functional option for overriding the creation +// time given to images. +func WithCreationTime(t v1.Time) Option { + return func(gbo *gobuildOpener) error { + gbo.creationTime = t + return nil + } +} + +// withBuilder is a functional option for overriding the way go binaries +// are built. This is exposed for testing. +func withBuilder(b builder) Option { + return func(gbo *gobuildOpener) error { + gbo.build = b + return nil + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/build/recorder.go b/vendor/github.com/google/go-containerregistry/pkg/ko/build/recorder.go new file mode 100644 index 00000000..3694eaa1 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/build/recorder.go @@ -0,0 +1,46 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Recorder composes with another Interface to record the built import paths. +type Recorder struct { + m sync.Mutex + ImportPaths []string + Builder Interface +} + +// Recorder implements Interface +var _ Interface = (*Recorder)(nil) + +// IsSupportedReference implements Interface +func (r *Recorder) IsSupportedReference(ip string) bool { + return r.Builder.IsSupportedReference(ip) +} + +// Build implements Interface +func (r *Recorder) Build(ip string) (v1.Image, error) { + func() { + r.m.Lock() + defer r.m.Unlock() + r.ImportPaths = append(r.ImportPaths, ip) + }() + return r.Builder.Build(ip) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/build/shared.go b/vendor/github.com/google/go-containerregistry/pkg/ko/build/shared.go new file mode 100644 index 00000000..72d0d3a9 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/build/shared.go @@ -0,0 +1,79 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Caching wraps a builder implementation in a layer that shares build results +// for the same inputs using a simple "future" implementation. Cached results +// may be invalidated by calling Invalidate with the same input passed to Build. +type Caching struct { + inner Interface + + m sync.Mutex + results map[string]*future +} + +// Caching implements Interface +var _ Interface = (*Caching)(nil) + +// NewCaching wraps the provided build.Interface in an implementation that +// shares build results for a given path until the result has been invalidated. +func NewCaching(inner Interface) (*Caching, error) { + return &Caching{ + inner: inner, + results: make(map[string]*future), + }, nil +} + +// Build implements Interface +func (c *Caching) Build(ip string) (v1.Image, error) { + f := func() *future { + // Lock the map of futures. + c.m.Lock() + defer c.m.Unlock() + + // If a future for "ip" exists, then return it. + f, ok := c.results[ip] + if ok { + return f + } + // Otherwise create and record a future for a Build of "ip". + f = newFuture(func() (v1.Image, error) { + return c.inner.Build(ip) + }) + c.results[ip] = f + return f + }() + + return f.Get() +} + +// IsSupportedReference implements Interface +func (c *Caching) IsSupportedReference(ip string) bool { + return c.inner.IsSupportedReference(ip) +} + +// Invalidate removes an import path's cached results. +func (c *Caching) Invalidate(ip string) { + c.m.Lock() + defer c.m.Unlock() + + delete(c.results, ip) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/publish/daemon.go b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/daemon.go new file mode 100644 index 00000000..d18e0703 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/daemon.go @@ -0,0 +1,80 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "fmt" + "log" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/daemon" +) + +const ( + // LocalDomain is a sentinel "registry" that represents side-loading images into the daemon. + LocalDomain = "ko.local" +) + +// demon is intentionally misspelled to avoid name collision (and drive Jon nuts). +type demon struct { + namer Namer + tags []string +} + +// NewDaemon returns a new publish.Interface that publishes images to a container daemon. +func NewDaemon(namer Namer, tags []string) Interface { + return &demon{namer, tags} +} + +// Publish implements publish.Interface +func (d *demon) Publish(img v1.Image, s string) (name.Reference, error) { + // https://github.com/google/go-containerregistry/issues/212 + s = strings.ToLower(s) + + h, err := img.Digest() + if err != nil { + return nil, err + } + + digestTag, err := name.NewTag(fmt.Sprintf("%s/%s:%s", LocalDomain, d.namer(s), h.Hex), name.WeakValidation) + if err != nil { + return nil, err + } + + log.Printf("Loading %v", digestTag) + if _, err := daemon.Write(digestTag, img); err != nil { + return nil, err + } + log.Printf("Loaded %v", digestTag) + + for _, tagName := range d.tags { + log.Printf("Adding tag %v", tagName) + tag, err := name.NewTag(fmt.Sprintf("%s/%s:%s", LocalDomain, d.namer(s), tagName), name.WeakValidation) + if err != nil { + return nil, err + } + + err = daemon.Tag(digestTag, tag) + + if err != nil { + return nil, err + } + log.Printf("Added tag %v", tagName) + } + + return &digestTag, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/publish/default.go b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/default.go new file mode 100644 index 00000000..0486b9ce --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/default.go @@ -0,0 +1,121 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "fmt" + "log" + "net/http" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// defalt is intentionally misspelled to avoid keyword collision (and drive Jon nuts). +type defalt struct { + base string + t http.RoundTripper + auth authn.Authenticator + namer Namer + tags []string +} + +// Option is a functional option for NewDefault. +type Option func(*defaultOpener) error + +type defaultOpener struct { + base string + t http.RoundTripper + auth authn.Authenticator + namer Namer + tags []string +} + +// Namer is a function from a supported import path to the portion of the resulting +// image name that follows the "base" repository name. +type Namer func(string) string + +// identity is the default namer, so import paths are affixed as-is under the repository +// name for maximum clarity, e.g. +// gcr.io/foo/github.com/bar/baz/cmd/blah +// ^--base--^ ^-------import path-------^ +func identity(in string) string { return in } + +// As some registries do not support pushing an image by digest, the default tag for pushing +// is the 'latest' tag. +var defaultTags = []string{"latest"} + +func (do *defaultOpener) Open() (Interface, error) { + return &defalt{ + base: do.base, + t: do.t, + auth: do.auth, + namer: do.namer, + tags: do.tags, + }, nil +} + +// NewDefault returns a new publish.Interface that publishes references under the provided base +// repository using the default keychain to authenticate and the default naming scheme. +func NewDefault(base string, options ...Option) (Interface, error) { + do := &defaultOpener{ + base: base, + t: http.DefaultTransport, + auth: authn.Anonymous, + namer: identity, + tags: defaultTags, + } + + for _, option := range options { + if err := option(do); err != nil { + return nil, err + } + } + return do.Open() +} + +// Publish implements publish.Interface +func (d *defalt) Publish(img v1.Image, s string) (name.Reference, error) { + // https://github.com/google/go-containerregistry/issues/212 + s = strings.ToLower(s) + + for _, tagName := range d.tags { + tag, err := name.NewTag(fmt.Sprintf("%s/%s:%s", d.base, d.namer(s), tagName), name.WeakValidation) + if err != nil { + return nil, err + } + + log.Printf("Publishing %v", tag) + // TODO: This is slow because we have to load the image multiple times. + // Figure out some way to publish the manifest with another tag. + if err := remote.Write(tag, img, d.auth, d.t); err != nil { + return nil, err + } + } + + h, err := img.Digest() + if err != nil { + return nil, err + } + dig, err := name.NewDigest(fmt.Sprintf("%s/%s@%s", d.base, d.namer(s), h), name.WeakValidation) + if err != nil { + return nil, err + } + log.Printf("Published %v", dig) + return &dig, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/publish/doc.go b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/doc.go new file mode 100644 index 00000000..835d575a --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package publish defines methods for publishing a v1.Image reference and +// returning the published digest for embedding back into a Kubernetes yaml. +package publish diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/publish/future.go b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/future.go new file mode 100644 index 00000000..15682755 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/future.go @@ -0,0 +1,75 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "sync" + + "github.com/google/go-containerregistry/pkg/name" +) + +func newFuture(work func() (name.Reference, error)) *future { + // Create a channel on which to send the result. + ch := make(chan *result) + // Initiate the actual work, sending its result + // along the above channel. + go func() { + ref, err := work() + ch <- &result{ref: ref, err: err} + }() + // Return a future for the above work. Callers should + // call .Get() on this result (as many times as needed). + // One of these calls will receive the result, store it, + // and close the channel so that the rest of the callers + // can consume it. + return &future{ + promise: ch, + } +} + +type result struct { + ref name.Reference + err error +} + +type future struct { + m sync.RWMutex + + result *result + promise chan *result +} + +// Get blocks on the result of the future. +func (f *future) Get() (name.Reference, error) { + // Block on the promise of a result until we get one. + result, ok := <-f.promise + if ok { + func() { + f.m.Lock() + defer f.m.Unlock() + // If we got the result, then store it so that + // others may access it. + f.result = result + // Close the promise channel so that others + // are signaled that the result is available. + close(f.promise) + }() + } + + f.m.RLock() + defer f.m.RUnlock() + + return f.result.ref, f.result.err +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/publish/options.go b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/options.go new file mode 100644 index 00000000..30c7b95d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/options.go @@ -0,0 +1,82 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "log" + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" +) + +// WithTransport is a functional option for overriding the default transport +// on a default publisher. +func WithTransport(t http.RoundTripper) Option { + return func(i *defaultOpener) error { + i.t = t + return nil + } +} + +// WithAuth is a functional option for overriding the default authenticator +// on a default publisher. +func WithAuth(auth authn.Authenticator) Option { + return func(i *defaultOpener) error { + i.auth = auth + return nil + } +} + +// WithAuthFromKeychain is a functional option for overriding the default +// authenticator on a default publisher using an authn.Keychain +func WithAuthFromKeychain(keys authn.Keychain) Option { + return func(i *defaultOpener) error { + // We parse this lazily because it is a repository prefix, which + // means that docker.io/mattmoor actually gets interpreted as + // docker.io/library/mattmoor, which gets tricky when we start + // appending things to it in the publisher. + repo, err := name.NewRepository(i.base, name.WeakValidation) + if err != nil { + return err + } + auth, err := keys.Resolve(repo.Registry) + if err != nil { + return err + } + if auth == authn.Anonymous { + log.Println("No matching credentials were found, falling back on anonymous") + } + i.auth = auth + return nil + } +} + +// WithNamer is a functional option for overriding the image naming behavior +// in our default publisher. +func WithNamer(n Namer) Option { + return func(i *defaultOpener) error { + i.namer = n + return nil + } +} + +// WithTags is a functional option for overriding the image tags +func WithTags(tags []string) Option { + return func(i *defaultOpener) error { + i.tags = tags + return nil + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/publish/publish.go b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/publish.go new file mode 100644 index 00000000..c6002ef9 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/publish.go @@ -0,0 +1,28 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Interface abstracts different methods for publishing images. +type Interface interface { + // Publish uploads the given v1.Image to a registry incorporating the + // provided string into the image's repository name. Returns the digest + // of the published image. + Publish(v1.Image, string) (name.Reference, error) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/publish/shared.go b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/shared.go new file mode 100644 index 00000000..edd2832b --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/publish/shared.go @@ -0,0 +1,76 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "sync" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// caching wraps a publisher implementation in a layer that shares publish results +// for the same inputs using a simple "future" implementation. +type caching struct { + inner Interface + + m sync.Mutex + results map[string]*entry +} + +// entry holds the last image published and the result of publishing it for a +// particular reference. +type entry struct { + img v1.Image + f *future +} + +// caching implements Interface +var _ Interface = (*caching)(nil) + +// NewCaching wraps the provided publish.Interface in an implementation that +// shares publish results for a given path until the passed image object changes. +func NewCaching(inner Interface) (Interface, error) { + return &caching{ + inner: inner, + results: make(map[string]*entry), + }, nil +} + +// Publish implements Interface +func (c *caching) Publish(img v1.Image, ref string) (name.Reference, error) { + f := func() *future { + // Lock the map of futures. + c.m.Lock() + defer c.m.Unlock() + + // If a future for "ref" exists, then return it. + ent, ok := c.results[ref] + if ok { + // If the image matches, then return the same future. + if ent.img == img { + return ent.f + } + } + // Otherwise create and record a future for publishing "img" to "ref". + f := newFuture(func() (name.Reference, error) { + return c.inner.Publish(img, ref) + }) + c.results[ref] = &entry{img: img, f: f} + return f + }() + + return f.Get() +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/resolve/doc.go b/vendor/github.com/google/go-containerregistry/pkg/ko/resolve/doc.go new file mode 100644 index 00000000..6d659452 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/resolve/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package resolve defines logic for resolving K8s yaml inputs to ko. +package resolve diff --git a/vendor/github.com/google/go-containerregistry/pkg/ko/resolve/resolve.go b/vendor/github.com/google/go-containerregistry/pkg/ko/resolve/resolve.go new file mode 100644 index 00000000..6a3ed56e --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/ko/resolve/resolve.go @@ -0,0 +1,160 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resolve + +import ( + "bytes" + "fmt" + "io" + "sync" + + yaml "gopkg.in/yaml.v2" + + "golang.org/x/sync/errgroup" + + "github.com/google/go-containerregistry/pkg/ko/build" + "github.com/google/go-containerregistry/pkg/ko/publish" +) + +// ImageReferences resolves supported references to images within the input yaml +// to published image digests. +func ImageReferences(input []byte, builder build.Interface, publisher publish.Interface) ([]byte, error) { + // First, walk the input objects and collect a list of supported references + refs := make(map[string]struct{}) + // The loop is to support multi-document yaml files. + // This is handled by using a yaml.Decoder and reading objects until io.EOF, see: + // https://github.com/go-yaml/yaml/blob/v2.2.1/yaml.go#L124 + decoder := yaml.NewDecoder(bytes.NewBuffer(input)) + for { + var obj interface{} + if err := decoder.Decode(&obj); err != nil { + if err == io.EOF { + break + } + return nil, err + } + // This simply returns the replaced object, which we discard during the gathering phase. + if _, err := replaceRecursive(obj, func(ref string) (string, error) { + if builder.IsSupportedReference(ref) { + refs[ref] = struct{}{} + } + return ref, nil + }); err != nil { + return nil, err + } + } + + // Next, perform parallel builds for each of the supported references. + var sm sync.Map + var errg errgroup.Group + for ref := range refs { + ref := ref + errg.Go(func() error { + img, err := builder.Build(ref) + if err != nil { + return err + } + digest, err := publisher.Publish(img, ref) + if err != nil { + return err + } + sm.Store(ref, digest.String()) + return nil + }) + } + if err := errg.Wait(); err != nil { + return nil, err + } + + // Last, walk the inputs again and replace the supported references with their published images. + decoder = yaml.NewDecoder(bytes.NewBuffer(input)) + buf := bytes.NewBuffer(nil) + encoder := yaml.NewEncoder(buf) + for { + var obj interface{} + if err := decoder.Decode(&obj); err != nil { + if err == io.EOF { + return buf.Bytes(), nil + } + return nil, err + } + // Recursively walk input, replacing supported reference with our computed digests. + obj2, err := replaceRecursive(obj, func(ref string) (string, error) { + if !builder.IsSupportedReference(ref) { + return ref, nil + } + if val, ok := sm.Load(ref); ok { + return val.(string), nil + } + return "", fmt.Errorf("resolved reference to %q not found", ref) + }) + if err != nil { + return nil, err + } + + if err := encoder.Encode(obj2); err != nil { + return nil, err + } + } +} + +type replaceString func(string) (string, error) + +// replaceRecursive walks the provided untyped object recursively by switching +// on the type of the object at each level. It supports walking through the +// keys and values of maps, and the elements of an array. When a leaf of type +// string is encountered, this will call the provided replaceString function on +// it. This function does not support walking through struct types, but also +// should not need to as the input is expected to be the result of parsing yaml +// or json into an interface{}, which should only produce primitives, maps and +// arrays. This function will return a copy of the object rebuilt by the walk +// with the replacements made. +func replaceRecursive(obj interface{}, rs replaceString) (interface{}, error) { + switch typed := obj.(type) { + case map[interface{}]interface{}: + m2 := make(map[interface{}]interface{}, len(typed)) + for k, v := range typed { + k2, err := replaceRecursive(k, rs) + if err != nil { + return nil, err + } + v2, err := replaceRecursive(v, rs) + if err != nil { + return nil, err + } + m2[k2] = v2 + } + return m2, nil + + case []interface{}: + a2 := make([]interface{}, len(typed)) + for idx, v := range typed { + v2, err := replaceRecursive(v, rs) + if err != nil { + return nil, err + } + a2[idx] = v2 + } + return a2, nil + + case string: + // call our replaceString on this string leaf. + return rs(typed) + + default: + // leave other leaves alone. + return typed, nil + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go new file mode 100644 index 00000000..aa574eb8 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -0,0 +1,56 @@ +package remote + +import ( + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +// CheckPushPermission returns an error if the given keychain cannot authorize +// a push operation to the given ref. +// +// This can be useful to check whether the caller has permission to push an +// image before doing work to construct the image. +// +// TODO(#412): Remove the need for this method. +func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { + auth, err := kc.Resolve(ref.Context().Registry) + if err != nil { + return err + } + + scopes := []string{ref.Scope(transport.PushScope)} + tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + if err != nil { + return err + } + // TODO(jasonhall): Against GCR, just doing the token handshake is + // enough, but this doesn't extend to Dockerhub + // (https://github.com/docker/hub-feedback/issues/1771), so we actually + // need to initiate an upload to tell whether the credentials can + // authorize a push. Figure out how to return early here when we can, + // to avoid a roundtrip for spec-compliant registries. + w := writer{ + ref: ref, + client: &http.Client{Transport: tr}, + } + loc, _, err := w.initiateUpload("", "") + if loc != "" { + // Since we're only initiating the upload to check whether we + // can, we should attempt to cancel it, in case initiating + // reserves some resources on the server. We shouldn't wait for + // cancelling to complete, and we don't care if it fails. + go w.cancelUpload(loc) + } + return err +} + +func (w *writer) cancelUpload(loc string) { + req, err := http.NewRequest(http.MethodDelete, loc, nil) + if err != nil { + return + } + _, _ = w.client.Do(req) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go new file mode 100644 index 00000000..9c570b7f --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -0,0 +1,271 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var defaultPlatform = v1.Platform{ + Architecture: "amd64", + OS: "linux", +} + +// ErrSchema1 indicates that we received a schema1 manifest from the registry. +// This library doesn't have plans to support this legacy image format: +// https://github.com/google/go-containerregistry/issues/377 +var ErrSchema1 = errors.New("unsupported MediaType: https://github.com/google/go-containerregistry/issues/377") + +// Descriptor provides access to metadata about remote artifact and accessors +// for efficiently converting it into a v1.Image or v1.ImageIndex. +type Descriptor struct { + fetcher + v1.Descriptor + Manifest []byte + + // So we can share this implementation with Image.. + platform v1.Platform +} + +type imageOpener struct { + auth authn.Authenticator + transport http.RoundTripper + ref name.Reference + client *http.Client + platform v1.Platform +} + +// Get returns a remote.Descriptor for the given reference. The response from +// the registry is left un-interpreted, for the most part. This is useful for +// querying what kind of artifact a reference represents. +func Get(ref name.Reference, options ...ImageOption) (*Descriptor, error) { + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + types.DockerManifestList, + types.OCIImageIndex, + // Just to look at them. + types.DockerManifestSchema1, + types.DockerManifestSchema1Signed, + } + return get(ref, acceptable, options...) +} + +// Handle options and fetch the manifest with the acceptable MediaTypes in the +// Accept header. +func get(ref name.Reference, acceptable []types.MediaType, options ...ImageOption) (*Descriptor, error) { + i := &imageOpener{ + auth: authn.Anonymous, + transport: http.DefaultTransport, + ref: ref, + platform: defaultPlatform, + } + + for _, option := range options { + if err := option(i); err != nil { + return nil, err + } + } + tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + if err != nil { + return nil, err + } + + f := fetcher{ + Ref: i.ref, + Client: &http.Client{Transport: tr}, + } + + b, desc, err := f.fetchManifest(ref, acceptable) + if err != nil { + return nil, err + } + + return &Descriptor{ + fetcher: f, + Manifest: b, + Descriptor: *desc, + platform: i.platform, + }, nil +} + +// Image converts the Descriptor into a v1.Image. +// +// If the fetched artifact is already an image, it will just return it. +// +// If the fetched artifact is an index, it will attempt to resolve the index to +// a child image with the appropriate platform. +// +// See WithPlatform to set the desired platform. +func (d *Descriptor) Image() (v1.Image, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, ErrSchema1 + case types.OCIImageIndex, types.DockerManifestList: + // We want an image but the registry has an index, resolve it to an image. + return d.remoteIndex().imageByPlatform(d.platform) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // These are expected. Enumerated here to allow a default case. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + + // Wrap the v1.Layers returned by this v1.Image in a hint for downstream + // remote.Write calls to facilitate cross-repo "mounting". + imgCore, err := partial.CompressedToImage(d.remoteImage()) + if err != nil { + return nil, err + } + return &mountableImage{ + Image: imgCore, + Reference: d.Ref, + }, nil +} + +// ImageIndex converts the Descriptor into a v1.ImageIndex. +func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, ErrSchema1 + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // We want an index but the registry has an image, nothing we can do. + return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType) + case types.OCIImageIndex, types.DockerManifestList: + // These are expected. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + return d.remoteIndex(), nil +} + +func (d *Descriptor) remoteImage() *remoteImage { + return &remoteImage{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +func (d *Descriptor) remoteIndex() *remoteIndex { + return &remoteIndex{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +// fetcher implements methods for reading from a registry. +type fetcher struct { + Ref name.Reference + Client *http.Client +} + +// url returns a url.Url for the specified path in the context of this remote image reference. +func (f *fetcher) url(resource, identifier string) url.URL { + return url.URL{ + Scheme: f.Ref.Context().Registry.Scheme(), + Host: f.Ref.Context().RegistryStr(), + Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), + } +} + +func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { + u := f.url("manifests", ref.Identifier()) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, nil, err + } + accept := []string{} + for _, mt := range acceptable { + accept = append(accept, string(mt)) + } + req.Header.Set("Accept", strings.Join(accept, ",")) + + resp, err := f.Client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, nil, err + } + + manifest, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + digest, size, err := v1.SHA256(bytes.NewReader(manifest)) + if err != nil { + return nil, nil, err + } + + mediaType := types.MediaType(resp.Header.Get("Content-Type")) + + // Validate the digest matches what we asked for, if pulling by digest. + if dgst, ok := ref.(name.Digest); ok { + if mediaType == types.DockerManifestSchema1Signed { + // Digests for this are stupid to calculate, ignore it. + } else if digest.String() != dgst.DigestStr() { + return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) + } + } else { + // Do nothing for tags; I give up. + // + // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, + // but so many registries implement this incorrectly that it's not worth checking. + // + // For reference: + // https://github.com/docker/distribution/issues/2395 + // https://github.com/GoogleContainerTools/kaniko/issues/298 + } + + // Return all this info since we have to calculate it anyway. + desc := v1.Descriptor{ + Digest: digest, + Size: size, + MediaType: mediaType, + } + + return manifest, &desc, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index 1be0ad2e..9f2d51e8 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -15,16 +15,11 @@ package remote import ( - "bytes" - "fmt" "io" "io/ioutil" "net/http" - "net/url" - "strings" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" @@ -33,11 +28,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/v1util" ) -var defaultPlatform = v1.Platform{ - Architecture: "amd64", - OS: "linux", -} - // remoteImage accesses an image from a remote registry type remoteImage struct { fetcher @@ -46,135 +36,27 @@ type remoteImage struct { configLock sync.Mutex // Protects config config []byte mediaType types.MediaType - platform v1.Platform } -// ImageOption is a functional option for Image. -type ImageOption func(*imageOpener) error - var _ partial.CompressedImageCore = (*remoteImage)(nil) -type imageOpener struct { - auth authn.Authenticator - transport http.RoundTripper - ref name.Reference - client *http.Client - platform v1.Platform -} - -func (i *imageOpener) Open() (v1.Image, error) { - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) - if err != nil { - return nil, err - } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - platform: i.platform, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: i.ref, - }, nil -} - // Image provides access to a remote image reference, applying functional options // to the underlying imageOpener before resolving the reference into a v1.Image. func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { - img := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, - platform: defaultPlatform, + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + // We resolve these to images later. + types.DockerManifestList, + types.OCIImageIndex, } - for _, option := range options { - if err := option(img); err != nil { - return nil, err - } - } - return img.Open() -} - -// fetcher implements methods for reading from a remote image. -type fetcher struct { - Ref name.Reference - Client *http.Client -} - -// url returns a url.Url for the specified path in the context of this remote image reference. -func (f *fetcher) url(resource, identifier string) url.URL { - return url.URL{ - Scheme: f.Ref.Context().Registry.Scheme(), - Host: f.Ref.Context().RegistryStr(), - Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), - } -} - -func (f *fetcher) fetchManifest(acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { - u := f.url("manifests", f.Ref.Identifier()) - req, err := http.NewRequest(http.MethodGet, u.String(), nil) + desc, err := get(ref, acceptable, options...) if err != nil { - return nil, nil, err - } - accept := []string{} - for _, mt := range acceptable { - accept = append(accept, string(mt)) - } - req.Header.Set("Accept", strings.Join(accept, ",")) - - resp, err := f.Client.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if err := transport.CheckError(resp, http.StatusOK); err != nil { - return nil, nil, err + return nil, err } - manifest, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - - digest, size, err := v1.SHA256(bytes.NewReader(manifest)) - if err != nil { - return nil, nil, err - } - - // Validate the digest matches what we asked for, if pulling by digest. - if dgst, ok := f.Ref.(name.Digest); ok { - if digest.String() != dgst.DigestStr() { - return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) - } - } else { - // Do nothing for tags; I give up. - // - // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, - // but so many registries implement this incorrectly that it's not worth checking. - // - // For reference: - // https://github.com/docker/distribution/issues/2395 - // https://github.com/GoogleContainerTools/kaniko/issues/298 - } - - // Return all this info since we have to calculate it anyway. - desc := v1.Descriptor{ - Digest: digest, - Size: size, - MediaType: types.MediaType(resp.Header.Get("Content-Type")), - } - - return manifest, &desc, nil + return desc.Image() } func (r *remoteImage) MediaType() (types.MediaType, error) { @@ -184,7 +66,6 @@ func (r *remoteImage) MediaType() (types.MediaType, error) { return types.DockerManifestSchema2, nil } -// TODO(jonjohnsonjr): Handle manifest lists. func (r *remoteImage) RawManifest() ([]byte, error) { r.manifestLock.Lock() defer r.manifestLock.Unlock() @@ -192,26 +73,18 @@ func (r *remoteImage) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteImage. acceptable := []types.MediaType{ types.DockerManifestSchema2, types.OCIManifestSchema1, - // We'll resolve these to an image based on the platform. - types.DockerManifestList, - types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } - // We want an image but the registry has an index, resolve it to an image. - for desc.MediaType == types.DockerManifestList || desc.MediaType == types.OCIImageIndex { - manifest, desc, err = r.matchImage(manifest) - if err != nil { - return nil, err - } - } - r.mediaType = desc.MediaType r.manifest = manifest return r.manifest, nil @@ -302,36 +175,3 @@ func (r *remoteImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) digest: h, }, nil } - -// This naively matches the first manifest with matching Architecture and OS. -// -// We should probably use this instead: -// github.com/containerd/containerd/platforms -// -// But first we'd need to migrate to: -// github.com/opencontainers/image-spec/specs-go/v1 -func (r *remoteImage) matchImage(rawIndex []byte) ([]byte, *v1.Descriptor, error) { - index, err := v1.ParseIndexManifest(bytes.NewReader(rawIndex)) - if err != nil { - return nil, nil, err - } - for _, childDesc := range index.Manifests { - // If platform is missing from child descriptor, assume it's amd64/linux. - p := defaultPlatform - if childDesc.Platform != nil { - p = *childDesc.Platform - } - if r.platform.Architecture == p.Architecture && r.platform.OS == p.OS { - childRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), childDesc.Digest), name.StrictValidation) - if err != nil { - return nil, nil, err - } - r.fetcher = fetcher{ - Client: r.Client, - Ref: childRef, - } - return r.fetchManifest([]types.MediaType{childDesc.MediaType}) - } - } - return nil, nil, fmt.Errorf("no matching image for %s/%s, index: %s", r.platform.Architecture, r.platform.OS, string(rawIndex)) -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index 03afc481..2cf9922c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -17,14 +17,11 @@ package remote import ( "bytes" "fmt" - "net/http" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -39,27 +36,17 @@ type remoteIndex struct { // Index provides access to a remote index reference, applying functional options // to the underlying imageOpener before resolving the reference into a v1.ImageIndex. func Index(ref name.Reference, options ...ImageOption) (v1.ImageIndex, error) { - i := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, + acceptable := []types.MediaType{ + types.DockerManifestList, + types.OCIImageIndex, } - for _, option := range options { - if err := option(i); err != nil { - return nil, err - } - } - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + desc, err := get(ref, acceptable, options...) if err != nil { return nil, err } - return &remoteIndex{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - }, nil + + return desc.ImageIndex() } func (r *remoteIndex) MediaType() (types.MediaType, error) { @@ -80,11 +67,14 @@ func (r *remoteIndex) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteIndex. acceptable := []types.MediaType{ types.DockerManifestList, types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } @@ -103,37 +93,93 @@ func (r *remoteIndex) IndexManifest() (*v1.IndexManifest, error) { } func (r *remoteIndex) Image(h v1.Hash) (v1.Image, error) { - imgRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: imgRef, - Client: r.Client, - }, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: r.Ref, - }, nil + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() } func (r *remoteIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { - idxRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - return &remoteIndex{ + return desc.ImageIndex() +} + +func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { + desc, err := r.childByPlatform(platform) + if err != nil { + return nil, err + } + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() +} + +// This naively matches the first manifest with matching Architecture and OS. +// +// We should probably use this instead: +// github.com/containerd/containerd/platforms +// +// But first we'd need to migrate to: +// github.com/opencontainers/image-spec/specs-go/v1 +func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + // If platform is missing from child descriptor, assume it's amd64/linux. + p := defaultPlatform + if childDesc.Platform != nil { + p = *childDesc.Platform + } + + if platform.Architecture == p.Architecture && platform.OS == p.OS { + return r.childDescriptor(childDesc, platform) + } + } + return nil, fmt.Errorf("no child with platform %s/%s in index %s", platform.Architecture, platform.OS, r.Ref) +} + +func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + if h == childDesc.Digest { + return r.childDescriptor(childDesc, defaultPlatform) + } + } + return nil, fmt.Errorf("no child with digest %s in index %s", h, r.Ref) +} + +func (r *remoteIndex) childRef(h v1.Hash) (name.Reference, error) { + return name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) +} + +// Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option. +func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) { + ref, err := r.childRef(child.Digest) + if err != nil { + return nil, err + } + manifest, desc, err := r.fetchManifest(ref, []types.MediaType{child.MediaType}) + if err != nil { + return nil, err + } + return &Descriptor{ fetcher: fetcher{ - Ref: idxRef, + Ref: ref, Client: r.Client, }, + Manifest: manifest, + Descriptor: *desc, + platform: platform, }, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 335e3fe5..1af76061 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -22,6 +22,9 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" ) +// ImageOption is a functional option for Image, index, and Get. +type ImageOption func(*imageOpener) error + // WithTransport is a functional option for overriding the default transport // on a remote image func WithTransport(t http.RoundTripper) ImageOption { @@ -56,6 +59,9 @@ func WithAuthFromKeychain(keys authn.Keychain) ImageOption { } } +// WithPlatform is a functional option for overriding the default platform +// that Image and Descriptor.Image use for resolving an index to an image. +// The default platform is amd64/linux. func WithPlatform(p v1.Platform) ImageOption { return func(i *imageOpener) error { i.platform = p diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 00256e8f..d1a7e2c8 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -15,6 +15,7 @@ package tarball import ( + "bytes" "compress/gzip" "io" "io/ioutil" @@ -103,6 +104,18 @@ func LayerFromOpener(opener Opener) (v1.Layer, error) { }, nil } +// LayerFromReader returns a v1.Layer given a io.Reader. +func LayerFromReader(reader io.Reader) (v1.Layer, error) { + // Buffering due to Opener requiring multiple calls. + a, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + return LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(a)), nil + }) +} + func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) { rc, err := opener() if err != nil { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go index 44dbe15a..2ee81f0b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go @@ -28,31 +28,41 @@ import ( // WriteToFile writes in the compressed format to a tarball, on disk. // This is just syntactic sugar wrapping tarball.Write with a new file. -func WriteToFile(p string, tag name.Tag, img v1.Image) error { +func WriteToFile(p string, ref name.Reference, img v1.Image) error { w, err := os.Create(p) if err != nil { return err } defer w.Close() - return Write(tag, img, w) + return Write(ref, img, w) } // MultiWriteToFile writes in the compressed format to a tarball, on disk. // This is just syntactic sugar wrapping tarball.MultiWrite with a new file. func MultiWriteToFile(p string, tagToImage map[name.Tag]v1.Image) error { + var refToImage map[name.Reference]v1.Image = make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWriteToFile(p, refToImage) +} + +// MultiRefWriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.MultiRefWrite with a new file. +func MultiRefWriteToFile(p string, refToImage map[name.Reference]v1.Image) error { w, err := os.Create(p) if err != nil { return err } defer w.Close() - return MultiWrite(tagToImage, w) + return MultiRefWrite(refToImage, w) } // Write is a wrapper to write a single image and tag to a tarball. -func Write(tag name.Tag, img v1.Image, w io.Writer) error { - return MultiWrite(map[name.Tag]v1.Image{tag: img}, w) +func Write(ref name.Reference, img v1.Image, w io.Writer) error { + return MultiRefWrite(map[name.Reference]v1.Image{ref: img}, w) } // MultiWrite writes the contents of each image to the provided reader, in the compressed format. @@ -61,10 +71,23 @@ func Write(tag name.Tag, img v1.Image, w io.Writer) error { // One file for each layer, named after the layer's SHA. // One file for the config blob, named after its SHA. func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer) error { + var refToImage map[name.Reference]v1.Image = make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWrite(refToImage, w) +} + +// MultiRefWrite writes the contents of each image to the provided reader, in the compressed format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One file for each layer, named after the layer's SHA. +// One file for the config blob, named after its SHA. +func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error { tf := tar.NewWriter(w) defer tf.Close() - imageToTags := dedupTagToImage(tagToImage) + imageToTags := dedupRefToImage(refToImage) var td tarDescriptor for img, tags := range imageToTags { @@ -135,14 +158,20 @@ func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer) error { return writeTarEntry(tf, "manifest.json", bytes.NewReader(tdBytes), int64(len(tdBytes))) } -func dedupTagToImage(tagToImage map[name.Tag]v1.Image) map[v1.Image][]string { +func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]string { imageToTags := make(map[v1.Image][]string) - for tag, img := range tagToImage { - if tags, ok := imageToTags[img]; ok { - imageToTags[img] = append(tags, tag.String()) + for ref, img := range refToImage { + if tag, ok := ref.(name.Tag); ok { + if tags, ok := imageToTags[img]; ok && tags != nil { + imageToTags[img] = append(tags, tag.String()) + } else { + imageToTags[img] = []string{tag.String()} + } } else { - imageToTags[img] = []string{tag.String()} + if _, ok := imageToTags[img]; !ok { + imageToTags[img] = nil + } } }