1
0
mirror of https://github.com/ko-build/ko.git synced 2025-07-15 23:54:17 +02:00

Bump ggcr depenency (#41)

* Update ggcr dependency

* Update kode

Fixed remote.Write interface change.

Drop name.WeakValidation (now default).
This commit is contained in:
jonjohnsonjr
2019-06-21 13:15:10 -07:00
committed by GitHub
parent 9bae8ab408
commit 116114f1f9
32 changed files with 522 additions and 142 deletions

5
Gopkg.lock generated
View File

@ -142,10 +142,11 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:907d8921af1cd16f38bc1ab77bce658830a4b46a1e293b627cfc1d7cc66e14b3" digest = "1:ea38e7f69c0eac433ce08e27804ab50e9f6b4454cab951ae7cb60d1b5f1e3e92"
name = "github.com/google/go-containerregistry" name = "github.com/google/go-containerregistry"
packages = [ packages = [
"pkg/authn", "pkg/authn",
"pkg/internal/retry",
"pkg/name", "pkg/name",
"pkg/v1", "pkg/v1",
"pkg/v1/daemon", "pkg/v1/daemon",
@ -161,7 +162,7 @@
"pkg/v1/v1util", "pkg/v1/v1util",
] ]
pruneopts = "UT" pruneopts = "UT"
revision = "7df26a6795af089f2cac25d48650d07960b2b33d" revision = "11836488c0a2f806544892ad2ff3ed07103e0768"
[[projects]] [[projects]]
branch = "master" branch = "master"

View File

@ -73,7 +73,7 @@ func init() {
} }
ref := viper.GetString("defaultBaseImage") ref := viper.GetString("defaultBaseImage")
dbi, err := name.ParseReference(ref, name.WeakValidation) dbi, err := name.ParseReference(ref)
if err != nil { if err != nil {
log.Fatalf("'defaultBaseImage': error parsing %q as image reference: %v", ref, err) log.Fatalf("'defaultBaseImage': error parsing %q as image reference: %v", ref, err)
} }
@ -82,7 +82,7 @@ func init() {
baseImageOverrides = make(map[string]name.Reference) baseImageOverrides = make(map[string]name.Reference)
overrides := viper.GetStringMapString("baseImageOverrides") overrides := viper.GetStringMapString("baseImageOverrides")
for k, v := range overrides { for k, v := range overrides {
bi, err := name.ParseReference(v, name.WeakValidation) bi, err := name.ParseReference(v)
if err != nil { if err != nil {
log.Fatalf("'baseImageOverrides': error parsing %q as image reference: %v", v, err) log.Fatalf("'baseImageOverrides': error parsing %q as image reference: %v", v, err)
} }

View File

@ -91,7 +91,7 @@ func makePublisher(no *options.NameOptions, lo *options.LocalOptions, ta *option
if repoName == "" { if repoName == "" {
return nil, errors.New("KO_DOCKER_REPO environment variable is unset") return nil, errors.New("KO_DOCKER_REPO environment variable is unset")
} }
_, err := name.NewRepository(repoName, name.WeakValidation) _, err := name.NewRepository(repoName)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to parse environment variable KO_DOCKER_REPO=%q as repository: %v", repoName, err) return nil, fmt.Errorf("failed to parse environment variable KO_DOCKER_REPO=%q as repository: %v", repoName, err)
} }

View File

@ -103,7 +103,7 @@ func (d *defalt) Publish(img v1.Image, s string) (name.Reference, error) {
log.Printf("Publishing %v", tag) log.Printf("Publishing %v", tag)
// TODO: This is slow because we have to load the image multiple times. // TODO: This is slow because we have to load the image multiple times.
// Figure out some way to publish the manifest with another tag. // Figure out some way to publish the manifest with another tag.
if err := remote.Write(tag, img, d.auth, d.t); err != nil { if err := remote.Write(tag, img, remote.WithAuth(d.auth), remote.WithTransport(d.t)); err != nil {
return nil, err return nil, err
} }
} }

View File

@ -1 +0,0 @@
../kenobi

View File

@ -0,0 +1,68 @@
// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package retry provides methods for retrying operations. It is a thin wrapper
// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier.
package retry
import (
"fmt"
"k8s.io/apimachinery/pkg/util/wait"
)
// This is implemented by several errors in the net package as well as our
// transport.Error.
type temporary interface {
Temporary() bool
}
// IsTemporary returns true if err implements Temporary() and it returns true.
func IsTemporary(err error) bool {
if te, ok := err.(temporary); ok && te.Temporary() {
return true
}
return false
}
// IsNotNil returns true if err is not nil.
func IsNotNil(err error) bool {
return err != nil
}
// Predicate determines whether an error should be retried.
type Predicate func(error) (retry bool)
// Retry retries a given function, f, until a predicate is satisfied, using
// exponential backoff. If the predicate is never satisfied, it will return the
// last error returned by f.
func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) {
if f == nil {
return fmt.Errorf("nil f passed to retry")
}
if p == nil {
return fmt.Errorf("nil p passed to retry")
}
condition := func() (bool, error) {
err = f()
if p(err) {
return false, nil
}
return true, err
}
wait.ExponentialBackoff(backoff, condition)
return
}

View File

@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// Package name defines structured types for representing image references.
package name package name
import ( import (

View File

@ -0,0 +1,42 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package name defines structured types for representing image references.
//
// What's in a name? For image references, not nearly enough!
//
// Image references look a lot like URLs, but they differ in that they don't
// contain the scheme (http or https), they can end with a :tag or a @digest
// (the latter being validated), and they perform defaulting for missing
// components.
//
// Since image references don't contain the scheme, we do our best to infer
// if we use http or https from the given hostname. We allow http fallback for
// any host that looks like localhost (localhost, 127.0.0.1, ::1), ends in
// ".local", or is in the "private" address space per RFC 1918. For everything
// else, we assume https only. To override this heuristic, use the Insecure
// option.
//
// Image references with a digest signal to us that we should verify the content
// of the image matches the digest. E.g. when pulling a Digest reference, we'll
// calculate the sha256 of the manifest returned by the registry and error out
// if it doesn't match what we asked for.
//
// For defaulting, we interpret "ubuntu" as
// "index.docker.io/library/ubuntu:latest" because we add the missing repo
// "library", the missing registry "index.docker.io", and the missing tag
// "latest". To disable this defaulting, use the StrictValidation option. This
// is useful e.g. to only allow image references that explicitly set a tag or
// digest, so that you don't accidentally pull "latest".
package name

View File

@ -19,6 +19,7 @@ import (
) )
// Image defines the interface for interacting with an OCI v1 image. // Image defines the interface for interacting with an OCI v1 image.
//go:generate counterfeiter -o fake/image.go . Image
type Image interface { type Image interface {
// Layers returns the ordered collection of filesystem layers that comprise this image. // Layers returns the ordered collection of filesystem layers that comprise this image.
// The order of the list is oldest/base layer first, and most-recent/top layer last. // The order of the list is oldest/base layer first, and most-recent/top layer last.

View File

@ -19,6 +19,7 @@ import (
) )
// ImageIndex defines the interface for interacting with an OCI image index. // ImageIndex defines the interface for interacting with an OCI image index.
//go:generate counterfeiter -o fake/index.go . ImageIndex
type ImageIndex interface { type ImageIndex interface {
// MediaType of this image's manifest. // MediaType of this image's manifest.
MediaType() (types.MediaType, error) MediaType() (types.MediaType, error)

View File

@ -16,6 +16,8 @@ package v1
import ( import (
"io" "io"
"github.com/google/go-containerregistry/pkg/v1/types"
) )
// Layer is an interface for accessing the properties of a particular layer of a v1.Image // Layer is an interface for accessing the properties of a particular layer of a v1.Image
@ -34,4 +36,7 @@ type Layer interface {
// Size returns the compressed size of the Layer. // Size returns the compressed size of the Layer.
Size() (int64, error) Size() (int64, error)
// MediaType returns the media type of the Layer.
MediaType() (types.MediaType, error)
} }

View File

@ -78,10 +78,11 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
cf.Config = cfg cf.Config = cfg
return configFile(base, cf) return ConfigFile(base, cf)
} }
func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { // ConfigFile mutates the provided v1.Image to have the provided v1.ConfigFile
func ConfigFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) {
m, err := base.Manifest() m, err := base.Manifest()
if err != nil { if err != nil {
return nil, err return nil, err
@ -106,7 +107,7 @@ func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) {
cfg := cf.DeepCopy() cfg := cf.DeepCopy()
cfg.Created = created cfg.Created = created
return configFile(base, cfg) return ConfigFile(base, cfg)
} }
type image struct { type image struct {
@ -476,7 +477,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) {
h.Created = v1.Time{Time: t} h.Created = v1.Time{Time: t}
} }
return configFile(newImage, cfg) return ConfigFile(newImage, cfg)
} }
func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) {
@ -555,5 +556,5 @@ func Canonical(img v1.Image) (v1.Image, error) {
cfg.ContainerConfig.Hostname = "" cfg.ContainerConfig.Hostname = ""
cfg.DockerVersion = "" cfg.DockerVersion = ""
return configFile(img, cfg) return ConfigFile(img, cfg)
} }

View File

@ -18,6 +18,7 @@ import (
"io" "io"
v1 "github.com/google/go-containerregistry/pkg/v1" v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/google/go-containerregistry/pkg/v1/v1util" "github.com/google/go-containerregistry/pkg/v1/v1util"
) )
@ -32,6 +33,9 @@ type CompressedLayer interface {
// Size returns the compressed size of the Layer. // Size returns the compressed size of the Layer.
Size() (int64, error) Size() (int64, error)
// Returns the mediaType for the compressed Layer
MediaType() (types.MediaType, error)
} }
// compressedLayerExtender implements v1.Image using the compressed base properties. // compressedLayerExtender implements v1.Image using the compressed base properties.

View File

@ -32,6 +32,9 @@ type UncompressedLayer interface {
// Uncompressed returns an io.ReadCloser for the uncompressed layer contents. // Uncompressed returns an io.ReadCloser for the uncompressed layer contents.
Uncompressed() (io.ReadCloser, error) Uncompressed() (io.ReadCloser, error)
// Returns the mediaType for the compressed Layer
MediaType() (types.MediaType, error)
} }
// uncompressedLayerExtender implements v1.Image using the uncompressed base properties. // uncompressedLayerExtender implements v1.Image using the uncompressed base properties.

View File

@ -22,6 +22,7 @@ import (
"io/ioutil" "io/ioutil"
v1 "github.com/google/go-containerregistry/pkg/v1" v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/google/go-containerregistry/pkg/v1/v1util" "github.com/google/go-containerregistry/pkg/v1/v1util"
) )
@ -80,6 +81,12 @@ func (cl *configLayer) Size() (int64, error) {
return int64(len(cl.content)), nil return int64(len(cl.content)), nil
} }
func (cl *configLayer) MediaType() (types.MediaType, error) {
// Defaulting this to OCIConfigJSON as it should remain
// backwards compatible with DockerConfigJSON
return types.OCIConfigJSON, nil
}
var _ v1.Layer = (*configLayer)(nil) var _ v1.Layer = (*configLayer)(nil)
// ConfigLayer implements v1.Layer from the raw config bytes. // ConfigLayer implements v1.Layer from the raw config bytes.

View File

@ -21,4 +21,5 @@ type Platform struct {
OSVersion string `json:"os.version,omitempty"` OSVersion string `json:"os.version,omitempty"`
OSFeatures []string `json:"os.features,omitempty"` OSFeatures []string `json:"os.features,omitempty"`
Variant string `json:"variant,omitempty"` Variant string `json:"variant,omitempty"`
Features []string `json:"features,omitempty"`
} }

View File

@ -45,6 +45,14 @@ func (ul *uncompressedLayer) Uncompressed() (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewBuffer(ul.content)), nil return ioutil.NopCloser(bytes.NewBuffer(ul.content)), nil
} }
// MediaType returns the media type of the layer
func (ul *uncompressedLayer) MediaType() (types.MediaType, error) {
// Technically the media type should be 'application/tar' but given that our
// v1.Layer doesn't force consumers to care about whether the layer is compressed
// we should be fine returning the DockerLayer media type
return types.DockerLayer, nil
}
var _ partial.UncompressedLayer = (*uncompressedLayer)(nil) var _ partial.UncompressedLayer = (*uncompressedLayer)(nil)
// Image returns a pseudo-randomly generated Image. // Image returns a pseudo-randomly generated Image.

View File

@ -1,6 +1,7 @@
package remote package remote
import ( import (
"fmt"
"net/http" "net/http"
"github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/authn"
@ -18,13 +19,13 @@ import (
func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error {
auth, err := kc.Resolve(ref.Context().Registry) auth, err := kc.Resolve(ref.Context().Registry)
if err != nil { if err != nil {
return err return fmt.Errorf("resolving authorization for %v failed: %v", ref.Context().Registry, err)
} }
scopes := []string{ref.Scope(transport.PushScope)} scopes := []string{ref.Scope(transport.PushScope)}
tr, err := transport.New(ref.Context().Registry, auth, t, scopes) tr, err := transport.New(ref.Context().Registry, auth, t, scopes)
if err != nil { if err != nil {
return err return fmt.Errorf("creating push check transport for %v failed: %v", ref.Context().Registry, err)
} }
// TODO(jasonhall): Against GCR, just doing the token handshake is // TODO(jasonhall): Against GCR, just doing the token handshake is
// enough, but this doesn't extend to Dockerhub // enough, but this doesn't extend to Dockerhub

View File

@ -20,15 +20,18 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/remote/transport"
) )
// Delete removes the specified image reference from the remote registry. // Delete removes the specified image reference from the remote registry.
func Delete(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) error { func Delete(ref name.Reference, options ...Option) error {
o, err := makeOptions(ref.Context().Registry, options...)
if err != nil {
return err
}
scopes := []string{ref.Scope(transport.DeleteScope)} scopes := []string{ref.Scope(transport.DeleteScope)}
tr, err := transport.New(ref.Context().Registry, auth, t, scopes) tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes)
if err != nil { if err != nil {
return err return err
} }

View File

@ -23,7 +23,6 @@ import (
"net/url" "net/url"
"strings" "strings"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1" v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/partial"
@ -52,18 +51,10 @@ type Descriptor struct {
platform v1.Platform platform v1.Platform
} }
type imageOpener struct {
auth authn.Authenticator
transport http.RoundTripper
ref name.Reference
client *http.Client
platform v1.Platform
}
// Get returns a remote.Descriptor for the given reference. The response from // Get returns a remote.Descriptor for the given reference. The response from
// the registry is left un-interpreted, for the most part. This is useful for // the registry is left un-interpreted, for the most part. This is useful for
// querying what kind of artifact a reference represents. // querying what kind of artifact a reference represents.
func Get(ref name.Reference, options ...ImageOption) (*Descriptor, error) { func Get(ref name.Reference, options ...Option) (*Descriptor, error) {
acceptable := []types.MediaType{ acceptable := []types.MediaType{
types.DockerManifestSchema2, types.DockerManifestSchema2,
types.OCIManifestSchema1, types.OCIManifestSchema1,
@ -78,26 +69,19 @@ func Get(ref name.Reference, options ...ImageOption) (*Descriptor, error) {
// Handle options and fetch the manifest with the acceptable MediaTypes in the // Handle options and fetch the manifest with the acceptable MediaTypes in the
// Accept header. // Accept header.
func get(ref name.Reference, acceptable []types.MediaType, options ...ImageOption) (*Descriptor, error) { func get(ref name.Reference, acceptable []types.MediaType, options ...Option) (*Descriptor, error) {
i := &imageOpener{ o, err := makeOptions(ref.Context().Registry, options...)
auth: authn.Anonymous, if err != nil {
transport: http.DefaultTransport, return nil, err
ref: ref,
platform: defaultPlatform,
} }
for _, option := range options { tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, []string{ref.Scope(transport.PullScope)})
if err := option(i); err != nil {
return nil, err
}
}
tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)})
if err != nil { if err != nil {
return nil, err return nil, err
} }
f := fetcher{ f := fetcher{
Ref: i.ref, Ref: ref,
Client: &http.Client{Transport: tr}, Client: &http.Client{Transport: tr},
} }
@ -110,7 +94,7 @@ func get(ref name.Reference, acceptable []types.MediaType, options ...ImageOptio
fetcher: f, fetcher: f,
Manifest: b, Manifest: b,
Descriptor: *desc, Descriptor: *desc,
platform: i.platform, platform: o.platform,
}, nil }, nil
} }
@ -241,12 +225,16 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType
} }
mediaType := types.MediaType(resp.Header.Get("Content-Type")) mediaType := types.MediaType(resp.Header.Get("Content-Type"))
contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest"))
if err == nil && mediaType == types.DockerManifestSchema1Signed {
// If we can parse the digest from the header, and it's a signed schema 1
// manifest, let's use that for the digest to appease older registries.
digest = contentDigest
}
// Validate the digest matches what we asked for, if pulling by digest. // Validate the digest matches what we asked for, if pulling by digest.
if dgst, ok := ref.(name.Digest); ok { if dgst, ok := ref.(name.Digest); ok {
if mediaType == types.DockerManifestSchema1Signed { if digest.String() != dgst.DigestStr() {
// Digests for this are stupid to calculate, ignore it.
} else if digest.String() != dgst.DigestStr() {
return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref)
} }
} else { } else {

View File

@ -15,6 +15,7 @@
package remote package remote
import ( import (
"fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -40,9 +41,8 @@ type remoteImage struct {
var _ partial.CompressedImageCore = (*remoteImage)(nil) var _ partial.CompressedImageCore = (*remoteImage)(nil)
// Image provides access to a remote image reference, applying functional options // Image provides access to a remote image reference.
// to the underlying imageOpener before resolving the reference into a v1.Image. func Image(ref name.Reference, options ...Option) (v1.Image, error) {
func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) {
acceptable := []types.MediaType{ acceptable := []types.MediaType{
types.DockerManifestSchema2, types.DockerManifestSchema2,
types.OCIManifestSchema1, types.OCIManifestSchema1,
@ -151,6 +151,22 @@ func (rl *remoteLayer) Manifest() (*v1.Manifest, error) {
return partial.Manifest(rl.ri) return partial.Manifest(rl.ri)
} }
// MediaType implements v1.Layer
func (rl *remoteLayer) MediaType() (types.MediaType, error) {
m, err := rl.Manifest()
if err != nil {
return "", err
}
for _, layer := range m.Layers {
if layer.Digest == rl.digest {
return layer.MediaType, nil
}
}
return "", fmt.Errorf("unable to find layer with digest: %v", rl.digest)
}
// Size implements partial.CompressedLayer // Size implements partial.CompressedLayer
func (rl *remoteLayer) Size() (int64, error) { func (rl *remoteLayer) Size() (int64, error) {
// Look up the size of this digest in the manifest to avoid a request. // Look up the size of this digest in the manifest to avoid a request.

View File

@ -33,9 +33,8 @@ type remoteIndex struct {
mediaType types.MediaType mediaType types.MediaType
} }
// Index provides access to a remote index reference, applying functional options // Index provides access to a remote index reference.
// to the underlying imageOpener before resolving the reference into a v1.ImageIndex. func Index(ref name.Reference, options ...Option) (v1.ImageIndex, error) {
func Index(ref name.Reference, options ...ImageOption) (v1.ImageIndex, error) {
acceptable := []types.MediaType{ acceptable := []types.MediaType{
types.DockerManifestList, types.DockerManifestList,
types.OCIImageIndex, types.OCIImageIndex,
@ -120,7 +119,7 @@ func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) {
return desc.Image() return desc.Image()
} }
// This naively matches the first manifest with matching Architecture and OS. // This naively matches the first manifest with matching platform attributes.
// //
// We should probably use this instead: // We should probably use this instead:
// github.com/containerd/containerd/platforms // github.com/containerd/containerd/platforms
@ -139,7 +138,7 @@ func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error)
p = *childDesc.Platform p = *childDesc.Platform
} }
if platform.Architecture == p.Architecture && platform.OS == p.OS { if matchesPlatform(p, platform) {
return r.childDescriptor(childDesc, platform) return r.childDescriptor(childDesc, platform)
} }
} }
@ -183,3 +182,49 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform)
platform: platform, platform: platform,
}, nil }, nil
} }
// matchesPlatform checks if the given platform matches the required platforms.
// The given platform matches the required platform if
// - architecture and OS are identical.
// - OS version and variant are identical if provided.
// - features and OS features of the required platform are subsets of those of the given platform.
func matchesPlatform(given, required v1.Platform) bool {
// Required fields that must be identical.
if given.Architecture != required.Architecture || given.OS != required.OS {
return false
}
// Optional fields that may be empty, but must be identical if provided.
if required.OSVersion != "" && given.OSVersion != required.OSVersion {
return false
}
if required.Variant != "" && given.Variant != required.Variant {
return false
}
// Verify required platform's features are a subset of given platform's features.
if !isSubset(given.OSFeatures, required.OSFeatures) {
return false
}
if !isSubset(given.Features, required.Features) {
return false
}
return true
}
// isSubset checks if the required array of strings is a subset of the given lst.
func isSubset(lst, required []string) bool {
set := make(map[string]bool)
for _, value := range lst {
set[value] = true
}
for _, value := range required {
if _, ok := set[value]; !ok {
return false
}
}
return true
}

View File

@ -20,7 +20,6 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/remote/transport"
) )
@ -30,10 +29,15 @@ type tags struct {
Tags []string `json:"tags"` Tags []string `json:"tags"`
} }
// List calls /tags/list for the given repository. // List calls /tags/list for the given repository, returning the list of tags
func List(repo name.Repository, auth authn.Authenticator, t http.RoundTripper) ([]string, error) { // in the "tags" property.
func List(repo name.Repository, options ...Option) ([]string, error) {
o, err := makeOptions(repo.Registry, options...)
if err != nil {
return nil, err
}
scopes := []string{repo.Scope(transport.PullScope)} scopes := []string{repo.Scope(transport.PullScope)}
tr, err := transport.New(repo.Registry, auth, t, scopes) tr, err := transport.New(repo.Registry, o.auth, o.transport, scopes)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -19,52 +19,92 @@ import (
"net/http" "net/http"
"github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1" v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
) )
// ImageOption is a functional option for Image, index, and Get. // Option is a functional option for remote operations.
type ImageOption func(*imageOpener) error type Option func(*options) error
type options struct {
auth authn.Authenticator
keychain authn.Keychain
transport http.RoundTripper
platform v1.Platform
}
func makeOptions(reg name.Registry, opts ...Option) (*options, error) {
o := &options{
auth: authn.Anonymous,
transport: http.DefaultTransport,
platform: defaultPlatform,
}
for _, option := range opts {
if err := option(o); err != nil {
return nil, err
}
}
if o.keychain != nil {
auth, err := o.keychain.Resolve(reg)
if err != nil {
return nil, err
}
if auth == authn.Anonymous {
log.Println("No matching credentials were found, falling back on anonymous")
}
o.auth = auth
}
// Wrap the transport in something that can retry network flakes.
o.transport = transport.NewRetry(o.transport)
return o, nil
}
// WithTransport is a functional option for overriding the default transport // WithTransport is a functional option for overriding the default transport
// on a remote image // for remote operations.
func WithTransport(t http.RoundTripper) ImageOption { //
return func(i *imageOpener) error { // The default transport its http.DefaultTransport.
i.transport = t func WithTransport(t http.RoundTripper) Option {
return func(o *options) error {
o.transport = t
return nil return nil
} }
} }
// WithAuth is a functional option for overriding the default authenticator // WithAuth is a functional option for overriding the default authenticator
// on a remote image // for remote operations.
func WithAuth(auth authn.Authenticator) ImageOption { //
return func(i *imageOpener) error { // The default authenticator is authn.Anonymous.
i.auth = auth func WithAuth(auth authn.Authenticator) Option {
return func(o *options) error {
o.auth = auth
return nil return nil
} }
} }
// WithAuthFromKeychain is a functional option for overriding the default // WithAuthFromKeychain is a functional option for overriding the default
// authenticator on a remote image using an authn.Keychain // authenticator for remote operations, using an authn.Keychain to find
func WithAuthFromKeychain(keys authn.Keychain) ImageOption { // credentials.
return func(i *imageOpener) error { //
auth, err := keys.Resolve(i.ref.Context().Registry) // The default authenticator is authn.Anonymous.
if err != nil { func WithAuthFromKeychain(keys authn.Keychain) Option {
return err return func(o *options) error {
} o.keychain = keys
if auth == authn.Anonymous {
log.Println("No matching credentials were found, falling back on anonymous")
}
i.auth = auth
return nil return nil
} }
} }
// WithPlatform is a functional option for overriding the default platform // WithPlatform is a functional option for overriding the default platform
// that Image and Descriptor.Image use for resolving an index to an image. // that Image and Descriptor.Image use for resolving an index to an image.
//
// The default platform is amd64/linux. // The default platform is amd64/linux.
func WithPlatform(p v1.Platform) ImageOption { func WithPlatform(p v1.Platform) Option {
return func(i *imageOpener) error { return func(o *options) error {
i.platform = p o.platform = p
return nil return nil
} }
} }

View File

@ -40,7 +40,7 @@ func (bt *basicTransport) RoundTrip(in *http.Request) (*http.Response, error) {
// we are redirected, only set it when the authorization header matches // we are redirected, only set it when the authorization header matches
// the host with which we are interacting. // the host with which we are interacting.
// In case of redirect http.Client can use an empty Host, check URL too. // In case of redirect http.Client can use an empty Host, check URL too.
if in.Host == bt.target || in.URL.Host == bt.target { if hdr != "" && (in.Host == bt.target || in.URL.Host == bt.target) {
in.Header.Set("Authorization", hdr) in.Header.Set("Authorization", hdr)
} }
in.Header.Set("User-Agent", transportName) in.Header.Set("User-Agent", transportName)

View File

@ -60,10 +60,14 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) {
// In case of redirect http.Client can use an empty Host, check URL too. // In case of redirect http.Client can use an empty Host, check URL too.
if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() { if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() {
in.Header.Set("Authorization", hdr) in.Header.Set("Authorization", hdr)
// When we ping() the registry, we determine whether to use http or https
// based on which scheme was successful. That is only valid for the
// registry server and not e.g. a separate token server or blob storage,
// so we should only override the scheme if the host is the registry.
in.URL.Scheme = bt.scheme
} }
in.Header.Set("User-Agent", transportName) in.Header.Set("User-Agent", transportName)
in.URL.Scheme = bt.scheme
return bt.inner.RoundTrip(in) return bt.inner.RoundTrip(in)
} }

View File

@ -48,6 +48,20 @@ func (e *Error) Error() string {
} }
} }
// Temporary returns whether the request that preceded the error is temporary.
func (e *Error) Temporary() bool {
if len(e.Errors) == 0 {
return false
}
for _, d := range e.Errors {
// TODO: Include other error types.
if d.Code != BlobUploadInvalidErrorCode {
return false
}
}
return true
}
// Diagnostic represents a single error returned by a Docker registry interaction. // Diagnostic represents a single error returned by a Docker registry interaction.
type Diagnostic struct { type Diagnostic struct {
Code ErrorCode `json:"code"` Code ErrorCode `json:"code"`

View File

@ -0,0 +1,89 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"net/http"
"time"
"github.com/google/go-containerregistry/pkg/internal/retry"
"k8s.io/apimachinery/pkg/util/wait"
)
// Sleep for 0.1, 0.3, 0.9, 2.7 seconds. This should cover networking blips.
var defaultBackoff = wait.Backoff{
Duration: 100 * time.Millisecond,
Factor: 3.0,
Jitter: 0.1,
Steps: 5,
}
var _ http.RoundTripper = (*retryTransport)(nil)
// retryTransport wraps a RoundTripper and retries temporary network errors.
type retryTransport struct {
inner http.RoundTripper
backoff wait.Backoff
predicate retry.Predicate
}
// Option is a functional option for retryTransport.
type Option func(*options)
type options struct {
backoff wait.Backoff
predicate retry.Predicate
}
// WithRetryBackoff sets the backoff for retry operations.
func WithRetryBackoff(backoff wait.Backoff) Option {
return func(o *options) {
o.backoff = backoff
}
}
// WithRetryPredicate sets the predicate for retry operations.
func WithRetryPredicate(predicate func(error) bool) Option {
return func(o *options) {
o.predicate = predicate
}
}
// NewRetry returns a transport that retries errors.
func NewRetry(inner http.RoundTripper, opts ...Option) http.RoundTripper {
o := &options{
backoff: defaultBackoff,
predicate: retry.IsTemporary,
}
for _, opt := range opts {
opt(o)
}
return &retryTransport{
inner: inner,
backoff: o.backoff,
predicate: o.predicate,
}
}
func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) {
roundtrip := func() error {
out, err = t.inner.RoundTrip(in)
return err
}
retry.Retry(roundtrip, t.predicate, t.backoff)
return
}

View File

@ -22,15 +22,16 @@ import (
"log" "log"
"net/http" "net/http"
"net/url" "net/url"
"time"
"github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/internal/retry"
"github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1" v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/remote/transport"
"github.com/google/go-containerregistry/pkg/v1/stream"
"github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/types"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
"k8s.io/apimachinery/pkg/util/wait"
) )
type manifest interface { type manifest interface {
@ -40,14 +41,19 @@ type manifest interface {
} }
// Write pushes the provided img to the specified image reference. // Write pushes the provided img to the specified image reference.
func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.RoundTripper) error { func Write(ref name.Reference, img v1.Image, options ...Option) error {
ls, err := img.Layers() ls, err := img.Layers()
if err != nil { if err != nil {
return err return err
} }
o, err := makeOptions(ref.Context().Registry, options...)
if err != nil {
return err
}
scopes := scopesForUploadingImage(ref, ls) scopes := scopesForUploadingImage(ref, ls)
tr, err := transport.New(ref.Context().Registry, auth, t, scopes) tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes)
if err != nil { if err != nil {
return err return err
} }
@ -57,17 +63,17 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro
} }
// Upload individual layers in goroutines and collect any errors. // Upload individual layers in goroutines and collect any errors.
// If we can dedupe by the layer digest, try to do so. If the layer is // If we can dedupe by the layer digest, try to do so. If we can't determine
// a stream.Layer, we can't dedupe and might re-upload. // the digest for whatever reason, we can't dedupe and might re-upload.
var g errgroup.Group var g errgroup.Group
uploaded := map[v1.Hash]bool{} uploaded := map[v1.Hash]bool{}
for _, l := range ls { for _, l := range ls {
l := l l := l
if _, ok := l.(*stream.Layer); !ok {
h, err := l.Digest() // Streaming layers calculate their digests while uploading them. Assume
if err != nil { // an error here indicates we need to upload the layer.
return err h, err := l.Digest()
} if err == nil {
// If we can determine the layer's digest ahead of // If we can determine the layer's digest ahead of
// time, use it to dedupe uploads. // time, use it to dedupe uploads.
if uploaded[h] { if uploaded[h] {
@ -81,14 +87,15 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro
}) })
} }
if l, err := partial.ConfigLayer(img); err == stream.ErrNotComputed { if l, err := partial.ConfigLayer(img); err != nil {
// We can't read the ConfigLayer, because of streaming layers, since the // We can't read the ConfigLayer, possibly because of streaming layers,
// config hasn't been calculated yet. // since the layer DiffIDs haven't been calculated yet. Attempt to wait
// for the other layers to be uploaded, then try the config again.
if err := g.Wait(); err != nil { if err := g.Wait(); err != nil {
return err return err
} }
// Now that all the layers are uploaded, upload the config file blob. // Now that all the layers are uploaded, try to upload the config file blob.
l, err := partial.ConfigLayer(img) l, err := partial.ConfigLayer(img)
if err != nil { if err != nil {
return err return err
@ -96,9 +103,6 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro
if err := w.uploadOne(l); err != nil { if err := w.uploadOne(l); err != nil {
return err return err
} }
} else if err != nil {
// This is an actual error, not a streaming error, just return it.
return err
} else { } else {
// We *can* read the ConfigLayer, so upload it concurrently with the layers. // We *can* read the ConfigLayer, so upload it concurrently with the layers.
g.Go(func() error { g.Go(func() error {
@ -285,19 +289,10 @@ func (w *writer) commitBlob(location, digest string) error {
// uploadOne performs a complete upload of a single layer. // uploadOne performs a complete upload of a single layer.
func (w *writer) uploadOne(l v1.Layer) error { func (w *writer) uploadOne(l v1.Layer) error {
var from, mount, digest string var from, mount string
if _, ok := l.(*stream.Layer); !ok { if h, err := l.Digest(); err == nil {
// Layer isn't streamable, we should take advantage of that to // If we know the digest, this isn't a streaming layer. Do an existence
// skip uploading if possible. // check so we can skip uploading the layer if possible.
// By sending ?digest= in the request, we'll also check that
// our computed digest matches the one computed by the
// registry.
h, err := l.Digest()
if err != nil {
return err
}
digest = h.String()
existing, err := w.checkExistingBlob(h) existing, err := w.checkExistingBlob(h)
if err != nil { if err != nil {
return err return err
@ -315,38 +310,50 @@ func (w *writer) uploadOne(l v1.Layer) error {
} }
} }
location, mounted, err := w.initiateUpload(from, mount) tryUpload := func() error {
if err != nil { location, mounted, err := w.initiateUpload(from, mount)
return err if err != nil {
} else if mounted { return err
} else if mounted {
h, err := l.Digest()
if err != nil {
return err
}
log.Printf("mounted blob: %s", h.String())
return nil
}
blob, err := l.Compressed()
if err != nil {
return err
}
location, err = w.streamBlob(blob, location)
if err != nil {
return err
}
h, err := l.Digest() h, err := l.Digest()
if err != nil { if err != nil {
return err return err
} }
log.Printf("mounted blob: %s", h.String()) digest := h.String()
if err := w.commitBlob(location, digest); err != nil {
return err
}
log.Printf("pushed blob: %s", digest)
return nil return nil
} }
blob, err := l.Compressed() // Try this three times, waiting 1s after first failure, 3s after second.
if err != nil { backoff := wait.Backoff{
return err Duration: 1.0 * time.Second,
} Factor: 3.0,
location, err = w.streamBlob(blob, location) Jitter: 0.1,
if err != nil { Steps: 3,
return err
} }
h, err := l.Digest() return retry.Retry(tryUpload, retry.IsTemporary, backoff)
if err != nil {
return err
}
digest = h.String()
if err := w.commitBlob(location, digest); err != nil {
return err
}
log.Printf("pushed blob: %s", digest)
return nil
} }
// commitImage does a PUT of the image's manifest. // commitImage does a PUT of the image's manifest.
@ -416,14 +423,18 @@ func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string {
// WriteIndex pushes the provided ImageIndex to the specified image reference. // WriteIndex pushes the provided ImageIndex to the specified image reference.
// WriteIndex will attempt to push all of the referenced manifests before // WriteIndex will attempt to push all of the referenced manifests before
// attempting to push the ImageIndex, to retain referential integrity. // attempting to push the ImageIndex, to retain referential integrity.
func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, t http.RoundTripper) error { func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error {
index, err := ii.IndexManifest() index, err := ii.IndexManifest()
if err != nil { if err != nil {
return err return err
} }
o, err := makeOptions(ref.Context().Registry, options...)
if err != nil {
return err
}
scopes := []string{ref.Scope(transport.PushScope)} scopes := []string{ref.Scope(transport.PushScope)}
tr, err := transport.New(ref.Context().Registry, auth, t, scopes) tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes)
if err != nil { if err != nil {
return err return err
} }
@ -453,7 +464,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator,
return err return err
} }
if err := WriteIndex(ref, ii, auth, t); err != nil { if err := WriteIndex(ref, ii, WithAuth(o.auth), WithTransport(o.transport)); err != nil {
return err return err
} }
case types.OCIManifestSchema1, types.DockerManifestSchema2: case types.OCIManifestSchema1, types.DockerManifestSchema2:
@ -461,7 +472,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator,
if err != nil { if err != nil {
return err return err
} }
if err := Write(ref, img, auth, t); err != nil { if err := Write(ref, img, WithAuth(o.auth), WithTransport(o.transport)); err != nil {
return err return err
} }
} }

View File

@ -24,6 +24,7 @@ import (
"sync" "sync"
v1 "github.com/google/go-containerregistry/pkg/v1" v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
) )
var ( var (
@ -81,6 +82,13 @@ func (l *Layer) Size() (int64, error) {
return l.size, nil return l.size, nil
} }
// MediaType implements v1.Layer
func (l *Layer) MediaType() (types.MediaType, error) {
// We return DockerLayer for now as uncompressed layers
// are unimplemented
return types.DockerLayer, nil
}
// Uncompressed implements v1.Layer. // Uncompressed implements v1.Layer.
func (l *Layer) Uncompressed() (io.ReadCloser, error) { func (l *Layer) Uncompressed() (io.ReadCloser, error) {
return nil, errors.New("NYI: stream.Layer.Uncompressed is not implemented") return nil, errors.New("NYI: stream.Layer.Uncompressed is not implemented")

View File

@ -226,6 +226,13 @@ func (ulft *uncompressedLayerFromTarball) Uncompressed() (io.ReadCloser, error)
return extractFileFromTar(ulft.opener, ulft.filePath) return extractFileFromTar(ulft.opener, ulft.filePath)
} }
func (ulft *uncompressedLayerFromTarball) MediaType() (types.MediaType, error) {
// Technically the media type should be 'application/tar' but given that our
// v1.Layer doesn't force consumers to care about whether the layer is compressed
// we should be fine returning the DockerLayer media type
return types.DockerLayer, nil
}
func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) {
cfg, err := partial.ConfigFile(i) cfg, err := partial.ConfigFile(i)
if err != nil { if err != nil {
@ -310,6 +317,11 @@ func (clft *compressedLayerFromTarball) Compressed() (io.ReadCloser, error) {
return extractFileFromTar(clft.opener, clft.filePath) return extractFileFromTar(clft.opener, clft.filePath)
} }
// MediaType implements partial.CompressedLayer
func (clft *compressedLayerFromTarball) MediaType() (types.MediaType, error) {
return types.DockerLayer, nil
}
// Size implements partial.CompressedLayer // Size implements partial.CompressedLayer
func (clft *compressedLayerFromTarball) Size() (int64, error) { func (clft *compressedLayerFromTarball) Size() (int64, error) {
r, err := clft.Compressed() r, err := clft.Compressed()

View File

@ -22,6 +22,7 @@ import (
"os" "os"
v1 "github.com/google/go-containerregistry/pkg/v1" v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/google/go-containerregistry/pkg/v1/v1util" "github.com/google/go-containerregistry/pkg/v1/v1util"
) )
@ -63,6 +64,10 @@ func (l *layer) Size() (int64, error) {
return l.size, nil return l.size, nil
} }
func (l *layer) MediaType() (types.MediaType, error) {
return types.DockerLayer, nil
}
// LayerFromFile returns a v1.Layer given a tarball // LayerFromFile returns a v1.Layer given a tarball
func LayerFromFile(path string) (v1.Layer, error) { func LayerFromFile(path string) (v1.Layer, error) {
opener := func() (io.ReadCloser, error) { opener := func() (io.ReadCloser, error) {