1
0
mirror of https://github.com/ko-build/ko.git synced 2024-12-12 08:54:09 +02:00

Bump ggcr depenency (#41)

* Update ggcr dependency

* Update kode

Fixed remote.Write interface change.

Drop name.WeakValidation (now default).
This commit is contained in:
jonjohnsonjr 2019-06-21 13:15:10 -07:00 committed by GitHub
parent 9bae8ab408
commit 116114f1f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 522 additions and 142 deletions

5
Gopkg.lock generated
View File

@ -142,10 +142,11 @@
[[projects]]
branch = "master"
digest = "1:907d8921af1cd16f38bc1ab77bce658830a4b46a1e293b627cfc1d7cc66e14b3"
digest = "1:ea38e7f69c0eac433ce08e27804ab50e9f6b4454cab951ae7cb60d1b5f1e3e92"
name = "github.com/google/go-containerregistry"
packages = [
"pkg/authn",
"pkg/internal/retry",
"pkg/name",
"pkg/v1",
"pkg/v1/daemon",
@ -161,7 +162,7 @@
"pkg/v1/v1util",
]
pruneopts = "UT"
revision = "7df26a6795af089f2cac25d48650d07960b2b33d"
revision = "11836488c0a2f806544892ad2ff3ed07103e0768"
[[projects]]
branch = "master"

View File

@ -73,7 +73,7 @@ func init() {
}
ref := viper.GetString("defaultBaseImage")
dbi, err := name.ParseReference(ref, name.WeakValidation)
dbi, err := name.ParseReference(ref)
if err != nil {
log.Fatalf("'defaultBaseImage': error parsing %q as image reference: %v", ref, err)
}
@ -82,7 +82,7 @@ func init() {
baseImageOverrides = make(map[string]name.Reference)
overrides := viper.GetStringMapString("baseImageOverrides")
for k, v := range overrides {
bi, err := name.ParseReference(v, name.WeakValidation)
bi, err := name.ParseReference(v)
if err != nil {
log.Fatalf("'baseImageOverrides': error parsing %q as image reference: %v", v, err)
}

View File

@ -91,7 +91,7 @@ func makePublisher(no *options.NameOptions, lo *options.LocalOptions, ta *option
if repoName == "" {
return nil, errors.New("KO_DOCKER_REPO environment variable is unset")
}
_, err := name.NewRepository(repoName, name.WeakValidation)
_, err := name.NewRepository(repoName)
if err != nil {
return nil, fmt.Errorf("failed to parse environment variable KO_DOCKER_REPO=%q as repository: %v", repoName, err)
}

View File

@ -103,7 +103,7 @@ func (d *defalt) Publish(img v1.Image, s string) (name.Reference, error) {
log.Printf("Publishing %v", tag)
// TODO: This is slow because we have to load the image multiple times.
// Figure out some way to publish the manifest with another tag.
if err := remote.Write(tag, img, d.auth, d.t); err != nil {
if err := remote.Write(tag, img, remote.WithAuth(d.auth), remote.WithTransport(d.t)); err != nil {
return nil, err
}
}

View File

@ -1 +0,0 @@
../kenobi

View File

@ -0,0 +1,68 @@
// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package retry provides methods for retrying operations. It is a thin wrapper
// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier.
package retry
import (
"fmt"
"k8s.io/apimachinery/pkg/util/wait"
)
// This is implemented by several errors in the net package as well as our
// transport.Error.
type temporary interface {
Temporary() bool
}
// IsTemporary returns true if err implements Temporary() and it returns true.
func IsTemporary(err error) bool {
if te, ok := err.(temporary); ok && te.Temporary() {
return true
}
return false
}
// IsNotNil returns true if err is not nil.
func IsNotNil(err error) bool {
return err != nil
}
// Predicate determines whether an error should be retried.
type Predicate func(error) (retry bool)
// Retry retries a given function, f, until a predicate is satisfied, using
// exponential backoff. If the predicate is never satisfied, it will return the
// last error returned by f.
func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) {
if f == nil {
return fmt.Errorf("nil f passed to retry")
}
if p == nil {
return fmt.Errorf("nil p passed to retry")
}
condition := func() (bool, error) {
err = f()
if p(err) {
return false, nil
}
return true, err
}
wait.ExponentialBackoff(backoff, condition)
return
}

View File

@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package name defines structured types for representing image references.
package name
import (

View File

@ -0,0 +1,42 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package name defines structured types for representing image references.
//
// What's in a name? For image references, not nearly enough!
//
// Image references look a lot like URLs, but they differ in that they don't
// contain the scheme (http or https), they can end with a :tag or a @digest
// (the latter being validated), and they perform defaulting for missing
// components.
//
// Since image references don't contain the scheme, we do our best to infer
// if we use http or https from the given hostname. We allow http fallback for
// any host that looks like localhost (localhost, 127.0.0.1, ::1), ends in
// ".local", or is in the "private" address space per RFC 1918. For everything
// else, we assume https only. To override this heuristic, use the Insecure
// option.
//
// Image references with a digest signal to us that we should verify the content
// of the image matches the digest. E.g. when pulling a Digest reference, we'll
// calculate the sha256 of the manifest returned by the registry and error out
// if it doesn't match what we asked for.
//
// For defaulting, we interpret "ubuntu" as
// "index.docker.io/library/ubuntu:latest" because we add the missing repo
// "library", the missing registry "index.docker.io", and the missing tag
// "latest". To disable this defaulting, use the StrictValidation option. This
// is useful e.g. to only allow image references that explicitly set a tag or
// digest, so that you don't accidentally pull "latest".
package name

View File

@ -19,6 +19,7 @@ import (
)
// Image defines the interface for interacting with an OCI v1 image.
//go:generate counterfeiter -o fake/image.go . Image
type Image interface {
// Layers returns the ordered collection of filesystem layers that comprise this image.
// The order of the list is oldest/base layer first, and most-recent/top layer last.

View File

@ -19,6 +19,7 @@ import (
)
// ImageIndex defines the interface for interacting with an OCI image index.
//go:generate counterfeiter -o fake/index.go . ImageIndex
type ImageIndex interface {
// MediaType of this image's manifest.
MediaType() (types.MediaType, error)

View File

@ -16,6 +16,8 @@ package v1
import (
"io"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// Layer is an interface for accessing the properties of a particular layer of a v1.Image
@ -34,4 +36,7 @@ type Layer interface {
// Size returns the compressed size of the Layer.
Size() (int64, error)
// MediaType returns the media type of the Layer.
MediaType() (types.MediaType, error)
}

View File

@ -78,10 +78,11 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
cf.Config = cfg
return configFile(base, cf)
return ConfigFile(base, cf)
}
func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) {
// ConfigFile mutates the provided v1.Image to have the provided v1.ConfigFile
func ConfigFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) {
m, err := base.Manifest()
if err != nil {
return nil, err
@ -106,7 +107,7 @@ func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) {
cfg := cf.DeepCopy()
cfg.Created = created
return configFile(base, cfg)
return ConfigFile(base, cfg)
}
type image struct {
@ -476,7 +477,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) {
h.Created = v1.Time{Time: t}
}
return configFile(newImage, cfg)
return ConfigFile(newImage, cfg)
}
func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) {
@ -555,5 +556,5 @@ func Canonical(img v1.Image) (v1.Image, error) {
cfg.ContainerConfig.Hostname = ""
cfg.DockerVersion = ""
return configFile(img, cfg)
return ConfigFile(img, cfg)
}

View File

@ -18,6 +18,7 @@ import (
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/google/go-containerregistry/pkg/v1/v1util"
)
@ -32,6 +33,9 @@ type CompressedLayer interface {
// Size returns the compressed size of the Layer.
Size() (int64, error)
// Returns the mediaType for the compressed Layer
MediaType() (types.MediaType, error)
}
// compressedLayerExtender implements v1.Image using the compressed base properties.

View File

@ -32,6 +32,9 @@ type UncompressedLayer interface {
// Uncompressed returns an io.ReadCloser for the uncompressed layer contents.
Uncompressed() (io.ReadCloser, error)
// Returns the mediaType for the compressed Layer
MediaType() (types.MediaType, error)
}
// uncompressedLayerExtender implements v1.Image using the uncompressed base properties.

View File

@ -22,6 +22,7 @@ import (
"io/ioutil"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/google/go-containerregistry/pkg/v1/v1util"
)
@ -80,6 +81,12 @@ func (cl *configLayer) Size() (int64, error) {
return int64(len(cl.content)), nil
}
func (cl *configLayer) MediaType() (types.MediaType, error) {
// Defaulting this to OCIConfigJSON as it should remain
// backwards compatible with DockerConfigJSON
return types.OCIConfigJSON, nil
}
var _ v1.Layer = (*configLayer)(nil)
// ConfigLayer implements v1.Layer from the raw config bytes.

View File

@ -21,4 +21,5 @@ type Platform struct {
OSVersion string `json:"os.version,omitempty"`
OSFeatures []string `json:"os.features,omitempty"`
Variant string `json:"variant,omitempty"`
Features []string `json:"features,omitempty"`
}

View File

@ -45,6 +45,14 @@ func (ul *uncompressedLayer) Uncompressed() (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewBuffer(ul.content)), nil
}
// MediaType returns the media type of the layer
func (ul *uncompressedLayer) MediaType() (types.MediaType, error) {
// Technically the media type should be 'application/tar' but given that our
// v1.Layer doesn't force consumers to care about whether the layer is compressed
// we should be fine returning the DockerLayer media type
return types.DockerLayer, nil
}
var _ partial.UncompressedLayer = (*uncompressedLayer)(nil)
// Image returns a pseudo-randomly generated Image.

View File

@ -1,6 +1,7 @@
package remote
import (
"fmt"
"net/http"
"github.com/google/go-containerregistry/pkg/authn"
@ -18,13 +19,13 @@ import (
func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error {
auth, err := kc.Resolve(ref.Context().Registry)
if err != nil {
return err
return fmt.Errorf("resolving authorization for %v failed: %v", ref.Context().Registry, err)
}
scopes := []string{ref.Scope(transport.PushScope)}
tr, err := transport.New(ref.Context().Registry, auth, t, scopes)
if err != nil {
return err
return fmt.Errorf("creating push check transport for %v failed: %v", ref.Context().Registry, err)
}
// TODO(jasonhall): Against GCR, just doing the token handshake is
// enough, but this doesn't extend to Dockerhub

View File

@ -20,15 +20,18 @@ import (
"net/http"
"net/url"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)
// Delete removes the specified image reference from the remote registry.
func Delete(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) error {
func Delete(ref name.Reference, options ...Option) error {
o, err := makeOptions(ref.Context().Registry, options...)
if err != nil {
return err
}
scopes := []string{ref.Scope(transport.DeleteScope)}
tr, err := transport.New(ref.Context().Registry, auth, t, scopes)
tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes)
if err != nil {
return err
}

View File

@ -23,7 +23,6 @@ import (
"net/url"
"strings"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
@ -52,18 +51,10 @@ type Descriptor struct {
platform v1.Platform
}
type imageOpener struct {
auth authn.Authenticator
transport http.RoundTripper
ref name.Reference
client *http.Client
platform v1.Platform
}
// Get returns a remote.Descriptor for the given reference. The response from
// the registry is left un-interpreted, for the most part. This is useful for
// querying what kind of artifact a reference represents.
func Get(ref name.Reference, options ...ImageOption) (*Descriptor, error) {
func Get(ref name.Reference, options ...Option) (*Descriptor, error) {
acceptable := []types.MediaType{
types.DockerManifestSchema2,
types.OCIManifestSchema1,
@ -78,26 +69,19 @@ func Get(ref name.Reference, options ...ImageOption) (*Descriptor, error) {
// Handle options and fetch the manifest with the acceptable MediaTypes in the
// Accept header.
func get(ref name.Reference, acceptable []types.MediaType, options ...ImageOption) (*Descriptor, error) {
i := &imageOpener{
auth: authn.Anonymous,
transport: http.DefaultTransport,
ref: ref,
platform: defaultPlatform,
func get(ref name.Reference, acceptable []types.MediaType, options ...Option) (*Descriptor, error) {
o, err := makeOptions(ref.Context().Registry, options...)
if err != nil {
return nil, err
}
for _, option := range options {
if err := option(i); err != nil {
return nil, err
}
}
tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)})
tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, []string{ref.Scope(transport.PullScope)})
if err != nil {
return nil, err
}
f := fetcher{
Ref: i.ref,
Ref: ref,
Client: &http.Client{Transport: tr},
}
@ -110,7 +94,7 @@ func get(ref name.Reference, acceptable []types.MediaType, options ...ImageOptio
fetcher: f,
Manifest: b,
Descriptor: *desc,
platform: i.platform,
platform: o.platform,
}, nil
}
@ -241,12 +225,16 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType
}
mediaType := types.MediaType(resp.Header.Get("Content-Type"))
contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest"))
if err == nil && mediaType == types.DockerManifestSchema1Signed {
// If we can parse the digest from the header, and it's a signed schema 1
// manifest, let's use that for the digest to appease older registries.
digest = contentDigest
}
// Validate the digest matches what we asked for, if pulling by digest.
if dgst, ok := ref.(name.Digest); ok {
if mediaType == types.DockerManifestSchema1Signed {
// Digests for this are stupid to calculate, ignore it.
} else if digest.String() != dgst.DigestStr() {
if digest.String() != dgst.DigestStr() {
return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref)
}
} else {

View File

@ -15,6 +15,7 @@
package remote
import (
"fmt"
"io"
"io/ioutil"
"net/http"
@ -40,9 +41,8 @@ type remoteImage struct {
var _ partial.CompressedImageCore = (*remoteImage)(nil)
// Image provides access to a remote image reference, applying functional options
// to the underlying imageOpener before resolving the reference into a v1.Image.
func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) {
// Image provides access to a remote image reference.
func Image(ref name.Reference, options ...Option) (v1.Image, error) {
acceptable := []types.MediaType{
types.DockerManifestSchema2,
types.OCIManifestSchema1,
@ -151,6 +151,22 @@ func (rl *remoteLayer) Manifest() (*v1.Manifest, error) {
return partial.Manifest(rl.ri)
}
// MediaType implements v1.Layer
func (rl *remoteLayer) MediaType() (types.MediaType, error) {
m, err := rl.Manifest()
if err != nil {
return "", err
}
for _, layer := range m.Layers {
if layer.Digest == rl.digest {
return layer.MediaType, nil
}
}
return "", fmt.Errorf("unable to find layer with digest: %v", rl.digest)
}
// Size implements partial.CompressedLayer
func (rl *remoteLayer) Size() (int64, error) {
// Look up the size of this digest in the manifest to avoid a request.

View File

@ -33,9 +33,8 @@ type remoteIndex struct {
mediaType types.MediaType
}
// Index provides access to a remote index reference, applying functional options
// to the underlying imageOpener before resolving the reference into a v1.ImageIndex.
func Index(ref name.Reference, options ...ImageOption) (v1.ImageIndex, error) {
// Index provides access to a remote index reference.
func Index(ref name.Reference, options ...Option) (v1.ImageIndex, error) {
acceptable := []types.MediaType{
types.DockerManifestList,
types.OCIImageIndex,
@ -120,7 +119,7 @@ func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) {
return desc.Image()
}
// This naively matches the first manifest with matching Architecture and OS.
// This naively matches the first manifest with matching platform attributes.
//
// We should probably use this instead:
// github.com/containerd/containerd/platforms
@ -139,7 +138,7 @@ func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error)
p = *childDesc.Platform
}
if platform.Architecture == p.Architecture && platform.OS == p.OS {
if matchesPlatform(p, platform) {
return r.childDescriptor(childDesc, platform)
}
}
@ -183,3 +182,49 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform)
platform: platform,
}, nil
}
// matchesPlatform checks if the given platform matches the required platforms.
// The given platform matches the required platform if
// - architecture and OS are identical.
// - OS version and variant are identical if provided.
// - features and OS features of the required platform are subsets of those of the given platform.
func matchesPlatform(given, required v1.Platform) bool {
// Required fields that must be identical.
if given.Architecture != required.Architecture || given.OS != required.OS {
return false
}
// Optional fields that may be empty, but must be identical if provided.
if required.OSVersion != "" && given.OSVersion != required.OSVersion {
return false
}
if required.Variant != "" && given.Variant != required.Variant {
return false
}
// Verify required platform's features are a subset of given platform's features.
if !isSubset(given.OSFeatures, required.OSFeatures) {
return false
}
if !isSubset(given.Features, required.Features) {
return false
}
return true
}
// isSubset checks if the required array of strings is a subset of the given lst.
func isSubset(lst, required []string) bool {
set := make(map[string]bool)
for _, value := range lst {
set[value] = true
}
for _, value := range required {
if _, ok := set[value]; !ok {
return false
}
}
return true
}

View File

@ -20,7 +20,6 @@ import (
"net/http"
"net/url"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)
@ -30,10 +29,15 @@ type tags struct {
Tags []string `json:"tags"`
}
// List calls /tags/list for the given repository.
func List(repo name.Repository, auth authn.Authenticator, t http.RoundTripper) ([]string, error) {
// List calls /tags/list for the given repository, returning the list of tags
// in the "tags" property.
func List(repo name.Repository, options ...Option) ([]string, error) {
o, err := makeOptions(repo.Registry, options...)
if err != nil {
return nil, err
}
scopes := []string{repo.Scope(transport.PullScope)}
tr, err := transport.New(repo.Registry, auth, t, scopes)
tr, err := transport.New(repo.Registry, o.auth, o.transport, scopes)
if err != nil {
return nil, err
}

View File

@ -19,52 +19,92 @@ import (
"net/http"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)
// ImageOption is a functional option for Image, index, and Get.
type ImageOption func(*imageOpener) error
// Option is a functional option for remote operations.
type Option func(*options) error
type options struct {
auth authn.Authenticator
keychain authn.Keychain
transport http.RoundTripper
platform v1.Platform
}
func makeOptions(reg name.Registry, opts ...Option) (*options, error) {
o := &options{
auth: authn.Anonymous,
transport: http.DefaultTransport,
platform: defaultPlatform,
}
for _, option := range opts {
if err := option(o); err != nil {
return nil, err
}
}
if o.keychain != nil {
auth, err := o.keychain.Resolve(reg)
if err != nil {
return nil, err
}
if auth == authn.Anonymous {
log.Println("No matching credentials were found, falling back on anonymous")
}
o.auth = auth
}
// Wrap the transport in something that can retry network flakes.
o.transport = transport.NewRetry(o.transport)
return o, nil
}
// WithTransport is a functional option for overriding the default transport
// on a remote image
func WithTransport(t http.RoundTripper) ImageOption {
return func(i *imageOpener) error {
i.transport = t
// for remote operations.
//
// The default transport its http.DefaultTransport.
func WithTransport(t http.RoundTripper) Option {
return func(o *options) error {
o.transport = t
return nil
}
}
// WithAuth is a functional option for overriding the default authenticator
// on a remote image
func WithAuth(auth authn.Authenticator) ImageOption {
return func(i *imageOpener) error {
i.auth = auth
// for remote operations.
//
// The default authenticator is authn.Anonymous.
func WithAuth(auth authn.Authenticator) Option {
return func(o *options) error {
o.auth = auth
return nil
}
}
// WithAuthFromKeychain is a functional option for overriding the default
// authenticator on a remote image using an authn.Keychain
func WithAuthFromKeychain(keys authn.Keychain) ImageOption {
return func(i *imageOpener) error {
auth, err := keys.Resolve(i.ref.Context().Registry)
if err != nil {
return err
}
if auth == authn.Anonymous {
log.Println("No matching credentials were found, falling back on anonymous")
}
i.auth = auth
// authenticator for remote operations, using an authn.Keychain to find
// credentials.
//
// The default authenticator is authn.Anonymous.
func WithAuthFromKeychain(keys authn.Keychain) Option {
return func(o *options) error {
o.keychain = keys
return nil
}
}
// WithPlatform is a functional option for overriding the default platform
// that Image and Descriptor.Image use for resolving an index to an image.
//
// The default platform is amd64/linux.
func WithPlatform(p v1.Platform) ImageOption {
return func(i *imageOpener) error {
i.platform = p
func WithPlatform(p v1.Platform) Option {
return func(o *options) error {
o.platform = p
return nil
}
}

View File

@ -40,7 +40,7 @@ func (bt *basicTransport) RoundTrip(in *http.Request) (*http.Response, error) {
// we are redirected, only set it when the authorization header matches
// the host with which we are interacting.
// In case of redirect http.Client can use an empty Host, check URL too.
if in.Host == bt.target || in.URL.Host == bt.target {
if hdr != "" && (in.Host == bt.target || in.URL.Host == bt.target) {
in.Header.Set("Authorization", hdr)
}
in.Header.Set("User-Agent", transportName)

View File

@ -60,10 +60,14 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) {
// In case of redirect http.Client can use an empty Host, check URL too.
if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() {
in.Header.Set("Authorization", hdr)
// When we ping() the registry, we determine whether to use http or https
// based on which scheme was successful. That is only valid for the
// registry server and not e.g. a separate token server or blob storage,
// so we should only override the scheme if the host is the registry.
in.URL.Scheme = bt.scheme
}
in.Header.Set("User-Agent", transportName)
in.URL.Scheme = bt.scheme
return bt.inner.RoundTrip(in)
}

View File

@ -48,6 +48,20 @@ func (e *Error) Error() string {
}
}
// Temporary returns whether the request that preceded the error is temporary.
func (e *Error) Temporary() bool {
if len(e.Errors) == 0 {
return false
}
for _, d := range e.Errors {
// TODO: Include other error types.
if d.Code != BlobUploadInvalidErrorCode {
return false
}
}
return true
}
// Diagnostic represents a single error returned by a Docker registry interaction.
type Diagnostic struct {
Code ErrorCode `json:"code"`

View File

@ -0,0 +1,89 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transport
import (
"net/http"
"time"
"github.com/google/go-containerregistry/pkg/internal/retry"
"k8s.io/apimachinery/pkg/util/wait"
)
// Sleep for 0.1, 0.3, 0.9, 2.7 seconds. This should cover networking blips.
var defaultBackoff = wait.Backoff{
Duration: 100 * time.Millisecond,
Factor: 3.0,
Jitter: 0.1,
Steps: 5,
}
var _ http.RoundTripper = (*retryTransport)(nil)
// retryTransport wraps a RoundTripper and retries temporary network errors.
type retryTransport struct {
inner http.RoundTripper
backoff wait.Backoff
predicate retry.Predicate
}
// Option is a functional option for retryTransport.
type Option func(*options)
type options struct {
backoff wait.Backoff
predicate retry.Predicate
}
// WithRetryBackoff sets the backoff for retry operations.
func WithRetryBackoff(backoff wait.Backoff) Option {
return func(o *options) {
o.backoff = backoff
}
}
// WithRetryPredicate sets the predicate for retry operations.
func WithRetryPredicate(predicate func(error) bool) Option {
return func(o *options) {
o.predicate = predicate
}
}
// NewRetry returns a transport that retries errors.
func NewRetry(inner http.RoundTripper, opts ...Option) http.RoundTripper {
o := &options{
backoff: defaultBackoff,
predicate: retry.IsTemporary,
}
for _, opt := range opts {
opt(o)
}
return &retryTransport{
inner: inner,
backoff: o.backoff,
predicate: o.predicate,
}
}
func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) {
roundtrip := func() error {
out, err = t.inner.RoundTrip(in)
return err
}
retry.Retry(roundtrip, t.predicate, t.backoff)
return
}

View File

@ -22,15 +22,16 @@ import (
"log"
"net/http"
"net/url"
"time"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/internal/retry"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
"github.com/google/go-containerregistry/pkg/v1/stream"
"github.com/google/go-containerregistry/pkg/v1/types"
"golang.org/x/sync/errgroup"
"k8s.io/apimachinery/pkg/util/wait"
)
type manifest interface {
@ -40,14 +41,19 @@ type manifest interface {
}
// Write pushes the provided img to the specified image reference.
func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.RoundTripper) error {
func Write(ref name.Reference, img v1.Image, options ...Option) error {
ls, err := img.Layers()
if err != nil {
return err
}
o, err := makeOptions(ref.Context().Registry, options...)
if err != nil {
return err
}
scopes := scopesForUploadingImage(ref, ls)
tr, err := transport.New(ref.Context().Registry, auth, t, scopes)
tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes)
if err != nil {
return err
}
@ -57,17 +63,17 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro
}
// Upload individual layers in goroutines and collect any errors.
// If we can dedupe by the layer digest, try to do so. If the layer is
// a stream.Layer, we can't dedupe and might re-upload.
// If we can dedupe by the layer digest, try to do so. If we can't determine
// the digest for whatever reason, we can't dedupe and might re-upload.
var g errgroup.Group
uploaded := map[v1.Hash]bool{}
for _, l := range ls {
l := l
if _, ok := l.(*stream.Layer); !ok {
h, err := l.Digest()
if err != nil {
return err
}
// Streaming layers calculate their digests while uploading them. Assume
// an error here indicates we need to upload the layer.
h, err := l.Digest()
if err == nil {
// If we can determine the layer's digest ahead of
// time, use it to dedupe uploads.
if uploaded[h] {
@ -81,14 +87,15 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro
})
}
if l, err := partial.ConfigLayer(img); err == stream.ErrNotComputed {
// We can't read the ConfigLayer, because of streaming layers, since the
// config hasn't been calculated yet.
if l, err := partial.ConfigLayer(img); err != nil {
// We can't read the ConfigLayer, possibly because of streaming layers,
// since the layer DiffIDs haven't been calculated yet. Attempt to wait
// for the other layers to be uploaded, then try the config again.
if err := g.Wait(); err != nil {
return err
}
// Now that all the layers are uploaded, upload the config file blob.
// Now that all the layers are uploaded, try to upload the config file blob.
l, err := partial.ConfigLayer(img)
if err != nil {
return err
@ -96,9 +103,6 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro
if err := w.uploadOne(l); err != nil {
return err
}
} else if err != nil {
// This is an actual error, not a streaming error, just return it.
return err
} else {
// We *can* read the ConfigLayer, so upload it concurrently with the layers.
g.Go(func() error {
@ -285,19 +289,10 @@ func (w *writer) commitBlob(location, digest string) error {
// uploadOne performs a complete upload of a single layer.
func (w *writer) uploadOne(l v1.Layer) error {
var from, mount, digest string
if _, ok := l.(*stream.Layer); !ok {
// Layer isn't streamable, we should take advantage of that to
// skip uploading if possible.
// By sending ?digest= in the request, we'll also check that
// our computed digest matches the one computed by the
// registry.
h, err := l.Digest()
if err != nil {
return err
}
digest = h.String()
var from, mount string
if h, err := l.Digest(); err == nil {
// If we know the digest, this isn't a streaming layer. Do an existence
// check so we can skip uploading the layer if possible.
existing, err := w.checkExistingBlob(h)
if err != nil {
return err
@ -315,38 +310,50 @@ func (w *writer) uploadOne(l v1.Layer) error {
}
}
location, mounted, err := w.initiateUpload(from, mount)
if err != nil {
return err
} else if mounted {
tryUpload := func() error {
location, mounted, err := w.initiateUpload(from, mount)
if err != nil {
return err
} else if mounted {
h, err := l.Digest()
if err != nil {
return err
}
log.Printf("mounted blob: %s", h.String())
return nil
}
blob, err := l.Compressed()
if err != nil {
return err
}
location, err = w.streamBlob(blob, location)
if err != nil {
return err
}
h, err := l.Digest()
if err != nil {
return err
}
log.Printf("mounted blob: %s", h.String())
digest := h.String()
if err := w.commitBlob(location, digest); err != nil {
return err
}
log.Printf("pushed blob: %s", digest)
return nil
}
blob, err := l.Compressed()
if err != nil {
return err
}
location, err = w.streamBlob(blob, location)
if err != nil {
return err
// Try this three times, waiting 1s after first failure, 3s after second.
backoff := wait.Backoff{
Duration: 1.0 * time.Second,
Factor: 3.0,
Jitter: 0.1,
Steps: 3,
}
h, err := l.Digest()
if err != nil {
return err
}
digest = h.String()
if err := w.commitBlob(location, digest); err != nil {
return err
}
log.Printf("pushed blob: %s", digest)
return nil
return retry.Retry(tryUpload, retry.IsTemporary, backoff)
}
// commitImage does a PUT of the image's manifest.
@ -416,14 +423,18 @@ func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string {
// WriteIndex pushes the provided ImageIndex to the specified image reference.
// WriteIndex will attempt to push all of the referenced manifests before
// attempting to push the ImageIndex, to retain referential integrity.
func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, t http.RoundTripper) error {
func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error {
index, err := ii.IndexManifest()
if err != nil {
return err
}
o, err := makeOptions(ref.Context().Registry, options...)
if err != nil {
return err
}
scopes := []string{ref.Scope(transport.PushScope)}
tr, err := transport.New(ref.Context().Registry, auth, t, scopes)
tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes)
if err != nil {
return err
}
@ -453,7 +464,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator,
return err
}
if err := WriteIndex(ref, ii, auth, t); err != nil {
if err := WriteIndex(ref, ii, WithAuth(o.auth), WithTransport(o.transport)); err != nil {
return err
}
case types.OCIManifestSchema1, types.DockerManifestSchema2:
@ -461,7 +472,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator,
if err != nil {
return err
}
if err := Write(ref, img, auth, t); err != nil {
if err := Write(ref, img, WithAuth(o.auth), WithTransport(o.transport)); err != nil {
return err
}
}

View File

@ -24,6 +24,7 @@ import (
"sync"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
)
var (
@ -81,6 +82,13 @@ func (l *Layer) Size() (int64, error) {
return l.size, nil
}
// MediaType implements v1.Layer
func (l *Layer) MediaType() (types.MediaType, error) {
// We return DockerLayer for now as uncompressed layers
// are unimplemented
return types.DockerLayer, nil
}
// Uncompressed implements v1.Layer.
func (l *Layer) Uncompressed() (io.ReadCloser, error) {
return nil, errors.New("NYI: stream.Layer.Uncompressed is not implemented")

View File

@ -226,6 +226,13 @@ func (ulft *uncompressedLayerFromTarball) Uncompressed() (io.ReadCloser, error)
return extractFileFromTar(ulft.opener, ulft.filePath)
}
func (ulft *uncompressedLayerFromTarball) MediaType() (types.MediaType, error) {
// Technically the media type should be 'application/tar' but given that our
// v1.Layer doesn't force consumers to care about whether the layer is compressed
// we should be fine returning the DockerLayer media type
return types.DockerLayer, nil
}
func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) {
cfg, err := partial.ConfigFile(i)
if err != nil {
@ -310,6 +317,11 @@ func (clft *compressedLayerFromTarball) Compressed() (io.ReadCloser, error) {
return extractFileFromTar(clft.opener, clft.filePath)
}
// MediaType implements partial.CompressedLayer
func (clft *compressedLayerFromTarball) MediaType() (types.MediaType, error) {
return types.DockerLayer, nil
}
// Size implements partial.CompressedLayer
func (clft *compressedLayerFromTarball) Size() (int64, error) {
r, err := clft.Compressed()

View File

@ -22,6 +22,7 @@ import (
"os"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/google/go-containerregistry/pkg/v1/v1util"
)
@ -63,6 +64,10 @@ func (l *layer) Size() (int64, error) {
return l.size, nil
}
func (l *layer) MediaType() (types.MediaType, error) {
return types.DockerLayer, nil
}
// LayerFromFile returns a v1.Layer given a tarball
func LayerFromFile(path string) (v1.Layer, error) {
opener := func() (io.ReadCloser, error) {