1
0
mirror of https://github.com/ko-build/ko.git synced 2024-12-03 08:35:34 +02:00

Add additional output formats (tarball and layout) (#134)

* Create a MultiPublisher

MultiPublisher mimics io.MultiWriter in that it will publish an image to
multiple publish.Interface implementations.

* Add publish.{Tarball,Layout}Publisher

This adds support for publishing in the tarball format and to an OCI
image layout.

The tarball format isn't great, yet. It only supports writing once
instead of appending.

* Consolidate options

These were spread all over the place for no reasons. Now all the
publisher related options are grouped together.

* Add options for tarball/layout

Adds --oci-layout-path, --tarball, and --push flags.

--push=false will disable the default behavior of publishing to a
registry.

* go mod vendor

* Add Close method to publish.Interface

This allows us to defer writing to the tarball until we've collected all
the images that have been published.

* Fix tests
This commit is contained in:
jonjohnsonjr 2020-02-19 09:30:01 -08:00 committed by GitHub
parent cfd680de28
commit 3c6a907da9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1197 additions and 98 deletions

View File

@ -30,10 +30,8 @@ import (
// addApply augments our CLI surface with apply.
func addApply(topLevel *cobra.Command) {
koApplyFlags := []string{}
lo := &options.LocalOptions{}
no := &options.NameOptions{}
po := &options.PublishOptions{}
fo := &options.FilenameOptions{}
ta := &options.TagsOptions{}
so := &options.SelectorOptions{}
sto := &options.StrictOptions{}
bo := &options.BuildOptions{}
@ -75,10 +73,11 @@ func addApply(topLevel *cobra.Command) {
if err != nil {
log.Fatalf("error creating builder: %v", err)
}
publisher, err := makePublisher(no, lo, ta)
publisher, err := makePublisher(po)
if err != nil {
log.Fatalf("error creating publisher: %v", err)
}
defer publisher.Close()
// Create a set of ko-specific flags to ignore when passing through
// kubectl global flags.
ignoreSet := make(map[string]struct{})
@ -145,10 +144,8 @@ func addApply(topLevel *cobra.Command) {
}
},
}
options.AddLocalArg(apply, lo)
options.AddNamingArgs(apply, no)
options.AddPublishArg(apply, po)
options.AddFileArg(apply, fo)
options.AddTagsArg(apply, ta)
options.AddSelectorArg(apply, so)
options.AddStrictArg(apply, sto)
options.AddBuildOptions(apply, bo)

View File

@ -30,10 +30,8 @@ import (
// addCreate augments our CLI surface with apply.
func addCreate(topLevel *cobra.Command) {
koCreateFlags := []string{}
lo := &options.LocalOptions{}
no := &options.NameOptions{}
po := &options.PublishOptions{}
fo := &options.FilenameOptions{}
ta := &options.TagsOptions{}
so := &options.SelectorOptions{}
sto := &options.StrictOptions{}
bo := &options.BuildOptions{}
@ -75,10 +73,11 @@ func addCreate(topLevel *cobra.Command) {
if err != nil {
log.Fatalf("error creating builder: %v", err)
}
publisher, err := makePublisher(no, lo, ta)
publisher, err := makePublisher(po)
if err != nil {
log.Fatalf("error creating publisher: %v", err)
}
defer publisher.Close()
// Create a set of ko-specific flags to ignore when passing through
// kubectl global flags.
ignoreSet := make(map[string]struct{})
@ -145,10 +144,8 @@ func addCreate(topLevel *cobra.Command) {
}
},
}
options.AddLocalArg(create, lo)
options.AddNamingArgs(create, no)
options.AddPublishArg(create, po)
options.AddFileArg(create, fo)
options.AddTagsArg(create, ta)
options.AddSelectorArg(create, so)
options.AddStrictArg(create, sto)
options.AddBuildOptions(create, bo)

View File

@ -23,18 +23,43 @@ import (
"github.com/spf13/cobra"
)
// NameOptions represents options for the ko binary.
type NameOptions struct {
// PublishOptions encapsulates options when publishing.
type PublishOptions struct {
Tags []string
// Push publishes images to a registry.
Push bool
// Local publishes images to a local docker daemon.
Local bool
InsecureRegistry bool
OCILayoutPath string
TarballFile string
// PreserveImportPaths preserves the full import path after KO_DOCKER_REPO.
PreserveImportPaths bool
// BaseImportPaths uses the base path without MD5 hash after KO_DOCKER_REPO.
BaseImportPaths bool
}
func AddNamingArgs(cmd *cobra.Command, no *NameOptions) {
cmd.Flags().BoolVarP(&no.PreserveImportPaths, "preserve-import-paths", "P", no.PreserveImportPaths,
func AddPublishArg(cmd *cobra.Command, po *PublishOptions) {
cmd.Flags().StringSliceVarP(&po.Tags, "tags", "t", []string{"latest"},
"Which tags to use for the produced image instead of the default 'latest' tag.")
cmd.Flags().BoolVar(&po.Push, "push", true, "Push images to KO_DOCKER_REPO")
cmd.Flags().BoolVarP(&po.Local, "local", "L", po.Local,
"Load into images to local docker daemon.")
cmd.Flags().BoolVar(&po.InsecureRegistry, "insecure-registry", po.InsecureRegistry,
"Whether to skip TLS verification on the registry")
cmd.Flags().StringVar(&po.OCILayoutPath, "oci-layout-path", "", "Path to save the OCI image layout of the built images")
cmd.Flags().StringVar(&po.TarballFile, "tarball", "", "File to save images tarballs")
cmd.Flags().BoolVarP(&po.PreserveImportPaths, "preserve-import-paths", "P", po.PreserveImportPaths,
"Whether to preserve the full import path after KO_DOCKER_REPO.")
cmd.Flags().BoolVarP(&no.BaseImportPaths, "base-import-paths", "B", no.BaseImportPaths,
cmd.Flags().BoolVarP(&po.BaseImportPaths, "base-import-paths", "B", po.BaseImportPaths,
"Whether to use the base path without MD5 hash after KO_DOCKER_REPO.")
}
@ -52,10 +77,10 @@ func baseImportPaths(importpath string) string {
return filepath.Base(importpath)
}
func MakeNamer(no *NameOptions) publish.Namer {
if no.PreserveImportPaths {
func MakeNamer(po *PublishOptions) publish.Namer {
if po.PreserveImportPaths {
return preserveImportPath
} else if no.BaseImportPaths {
} else if po.BaseImportPaths {
return baseImportPaths
}
return packageWithMD5

View File

@ -24,9 +24,7 @@ import (
// addPublish augments our CLI surface with publish.
func addPublish(topLevel *cobra.Command) {
lo := &options.LocalOptions{}
no := &options.NameOptions{}
ta := &options.TagsOptions{}
po := &options.PublishOptions{}
bo := &options.BuildOptions{}
publish := &cobra.Command{
@ -64,10 +62,11 @@ func addPublish(topLevel *cobra.Command) {
if err != nil {
log.Fatalf("error creating builder: %v", err)
}
publisher, err := makePublisher(no, lo, ta)
publisher, err := makePublisher(po)
if err != nil {
log.Fatalf("error creating publisher: %v", err)
}
defer publisher.Close()
ctx := createCancellableContext()
images, err := publishImages(ctx, args, publisher, builder)
if err != nil {
@ -78,9 +77,7 @@ func addPublish(topLevel *cobra.Command) {
}
},
}
options.AddLocalArg(publish, lo)
options.AddNamingArgs(publish, no)
options.AddTagsArg(publish, ta)
options.AddPublishArg(publish, po)
options.AddBuildOptions(publish, bo)
topLevel.AddCommand(publish)
}

View File

@ -24,10 +24,8 @@ import (
// addResolve augments our CLI surface with resolve.
func addResolve(topLevel *cobra.Command) {
lo := &options.LocalOptions{}
no := &options.NameOptions{}
po := &options.PublishOptions{}
fo := &options.FilenameOptions{}
ta := &options.TagsOptions{}
so := &options.SelectorOptions{}
sto := &options.StrictOptions{}
bo := &options.BuildOptions{}
@ -62,20 +60,19 @@ func addResolve(topLevel *cobra.Command) {
if err != nil {
log.Fatalf("error creating builder: %v", err)
}
publisher, err := makePublisher(no, lo, ta)
publisher, err := makePublisher(po)
if err != nil {
log.Fatalf("error creating publisher: %v", err)
}
defer publisher.Close()
ctx := createCancellableContext()
if err := resolveFilesToWriter(ctx, builder, publisher, fo, so, sto, os.Stdout); err != nil {
log.Fatal(err)
}
},
}
options.AddLocalArg(resolve, lo)
options.AddNamingArgs(resolve, no)
options.AddPublishArg(resolve, po)
options.AddFileArg(resolve, fo)
options.AddTagsArg(resolve, ta)
options.AddSelectorArg(resolve, so)
options.AddStrictArg(resolve, sto)
options.AddBuildOptions(resolve, bo)

View File

@ -111,16 +111,19 @@ func makeBuilder(bo *options.BuildOptions) (*build.Caching, error) {
return build.NewCaching(innerBuilder)
}
func makePublisher(no *options.NameOptions, lo *options.LocalOptions, ta *options.TagsOptions) (publish.Interface, error) {
func makePublisher(po *options.PublishOptions) (publish.Interface, error) {
// Create the publish.Interface that we will use to publish image references
// to either a docker daemon or a container image registry.
innerPublisher, err := func() (publish.Interface, error) {
namer := options.MakeNamer(no)
repoName := os.Getenv("KO_DOCKER_REPO")
if lo.Local || repoName == publish.LocalDomain {
return publish.NewDaemon(namer, ta.Tags), nil
namer := options.MakeNamer(po)
if repoName == publish.LocalDomain || po.Local {
// TODO(jonjohnsonjr): I'm assuming that nobody will
// use local with other publishers, but that might
// not be true.
return publish.NewDaemon(namer, po.Tags), nil
}
if repoName == "" {
return nil, errors.New("KO_DOCKER_REPO environment variable is unset")
}
@ -130,12 +133,31 @@ func makePublisher(no *options.NameOptions, lo *options.LocalOptions, ta *option
}
}
return publish.NewDefault(repoName,
publish.WithTransport(defaultTransport()),
publish.WithAuthFromKeychain(authn.DefaultKeychain),
publish.WithNamer(namer),
publish.WithTags(ta.Tags),
publish.Insecure(lo.InsecureRegistry))
publishers := []publish.Interface{}
if po.OCILayoutPath != "" {
lp, err := publish.NewLayout(po.OCILayoutPath)
if err != nil {
return nil, fmt.Errorf("failed to create LayoutPublisher for %q: %v", po.OCILayoutPath, err)
}
publishers = append(publishers, lp)
}
if po.TarballFile != "" {
tp := publish.NewTarball(po.TarballFile, repoName, namer, po.Tags)
publishers = append(publishers, tp)
}
if po.Push {
dp, err := publish.NewDefault(repoName,
publish.WithTransport(defaultTransport()),
publish.WithAuthFromKeychain(authn.DefaultKeychain),
publish.WithNamer(namer),
publish.WithTags(po.Tags),
publish.Insecure(po.InsecureRegistry))
if err != nil {
return nil, err
}
publishers = append(publishers, dp)
}
return publish.MultiPublisher(publishers...), nil
}()
if err != nil {
return nil, err

View File

@ -27,9 +27,7 @@ import (
// addRun augments our CLI surface with run.
func addRun(topLevel *cobra.Command) {
lo := &options.LocalOptions{}
no := &options.NameOptions{}
ta := &options.TagsOptions{}
po := &options.PublishOptions{}
bo := &options.BuildOptions{}
run := &cobra.Command{
@ -69,10 +67,11 @@ func addRun(topLevel *cobra.Command) {
if err != nil {
log.Fatalf("error creating builder: %v", err)
}
publisher, err := makePublisher(no, lo, ta)
publisher, err := makePublisher(po)
if err != nil {
log.Fatalf("error creating publisher: %v", err)
}
defer publisher.Close()
if len(os.Args) < 3 {
log.Fatalf("usage: %s run <package>", os.Args[0])
@ -137,9 +136,7 @@ func addRun(topLevel *cobra.Command) {
UnknownFlags: true,
},
}
options.AddLocalArg(run, lo)
options.AddNamingArgs(run, no)
options.AddTagsArg(run, ta)
options.AddPublishArg(run, po)
options.AddBuildOptions(run, bo)
topLevel.AddCommand(run)

View File

@ -72,6 +72,10 @@ func (f *fixedPublish) Publish(_ v1.Image, s string) (name.Reference, error) {
return &d, nil
}
func (f *fixedPublish) Close() error {
return nil
}
func ComputeDigest(base name.Repository, ref string, h v1.Hash) string {
d, err := name.NewDigest(fmt.Sprintf("%s/%s@%s", base, ref, h))
if err != nil {

View File

@ -78,3 +78,7 @@ func (d *demon) Publish(img v1.Image, s string) (name.Reference, error) {
return &digestTag, nil
}
func (d *demon) Close() error {
return nil
}

View File

@ -140,3 +140,7 @@ func (d *defalt) Publish(img v1.Image, s string) (name.Reference, error) {
log.Printf("Published %v", dig)
return &dig, nil
}
func (d *defalt) Close() error {
return nil
}

66
pkg/publish/layout.go Normal file
View File

@ -0,0 +1,66 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package publish
import (
"fmt"
"log"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/layout"
)
type LayoutPublisher struct {
p layout.Path
}
// NewLayout returns a new publish.Interface that saves images to an OCI Image Layout.
func NewLayout(path string) (Interface, error) {
p, err := layout.FromPath(path)
if err != nil {
p, err = layout.Write(path, empty.Index)
if err != nil {
return nil, err
}
}
return &LayoutPublisher{p}, nil
}
// Publish implements publish.Interface.
func (l *LayoutPublisher) Publish(img v1.Image, s string) (name.Reference, error) {
log.Printf("Saving %v", s)
if err := l.p.AppendImage(img); err != nil {
return nil, err
}
log.Printf("Saved %v", s)
h, err := img.Digest()
if err != nil {
return nil, err
}
dig, err := name.NewDigest(fmt.Sprintf("%s@%s", l.p, h))
if err != nil {
return nil, err
}
return dig, nil
}
func (l *LayoutPublisher) Close() error {
return nil
}

View File

@ -0,0 +1,48 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package publish
import (
"io/ioutil"
"os"
"strings"
"testing"
"github.com/google/go-containerregistry/pkg/v1/random"
)
func TestLayout(t *testing.T) {
img, err := random.Image(1024, 1)
if err != nil {
t.Fatalf("random.Image() = %v", err)
}
importpath := "github.com/Google/go-containerregistry/cmd/crane"
tmp, err := ioutil.TempDir("", "ko")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
lp, err := NewLayout(tmp)
if err != nil {
t.Errorf("NewLayout() = %v", err)
}
if d, err := lp.Publish(img, importpath); err != nil {
t.Errorf("Publish() = %v", err)
} else if !strings.HasPrefix(d.String(), tmp) {
t.Errorf("Publish() = %v, wanted prefix %v", d, tmp)
}
}

54
pkg/publish/multi.go Normal file
View File

@ -0,0 +1,54 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package publish
import (
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// MultiPublisher creates a publisher that publishes to all
// the provided publishers, similar to the Unix tee(1) command.
//
// When calling Publish, the name.Reference returned will be the return value
// of the last publisher passed to MultiPublisher (last one wins).
func MultiPublisher(publishers ...Interface) Interface {
return &multiPublisher{publishers}
}
type multiPublisher struct {
publishers []Interface
}
// Publish implements publish.Interface.
func (p *multiPublisher) Publish(img v1.Image, s string) (ref name.Reference, err error) {
for _, pub := range p.publishers {
ref, err = pub.Publish(img, s)
if err != nil {
return
}
}
return
}
func (p *multiPublisher) Close() (err error) {
for _, pub := range p.publishers {
if perr := pub.Close(); perr != nil {
err = perr
}
}
return
}

63
pkg/publish/multi_test.go Normal file
View File

@ -0,0 +1,63 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package publish
import (
"fmt"
"io/ioutil"
"os"
"testing"
"github.com/google/go-containerregistry/pkg/v1/random"
)
func TestMulti(t *testing.T) {
img, err := random.Image(1024, 1)
if err != nil {
t.Fatalf("random.Image() = %v", err)
}
base := "blah"
repoName := fmt.Sprintf("%s/%s", "example.com", base)
importpath := "github.com/Google/go-containerregistry/cmd/crane"
fp, err := ioutil.TempFile("", "")
if err != nil {
t.Fatal(err)
}
defer fp.Close()
defer os.Remove(fp.Name())
tp := NewTarball(fp.Name(), repoName, md5Hash, []string{})
tmp, err := ioutil.TempDir("", "ko")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
lp, err := NewLayout(tmp)
if err != nil {
t.Errorf("NewLayout() = %v", err)
}
p := MultiPublisher(lp, tp)
if _, err := p.Publish(img, importpath); err != nil {
t.Errorf("Publish() = %v", err)
}
if err := p.Close(); err != nil {
t.Errorf("Close() = %v", err)
}
}

View File

@ -25,4 +25,8 @@ type Interface interface {
// provided string into the image's repository name. Returns the digest
// of the published image.
Publish(v1.Image, string) (name.Reference, error)
// Close exists for the tarball implementation so we can
// do the whole thing in one write.
Close() error
}

View File

@ -74,3 +74,7 @@ func (c *caching) Publish(img v1.Image, ref string) (name.Reference, error) {
return f.Get()
}
func (c *caching) Close() error {
return c.inner.Close()
}

View File

@ -30,17 +30,21 @@ type slowpublish struct {
// slowpublish implements Interface
var _ Interface = (*slowpublish)(nil)
func (sb *slowpublish) Publish(img v1.Image, ref string) (name.Reference, error) {
time.Sleep(sb.sleep)
func (sp *slowpublish) Publish(img v1.Image, ref string) (name.Reference, error) {
time.Sleep(sp.sleep)
return makeRef()
}
func (sp *slowpublish) Close() error {
return nil
}
func TestCaching(t *testing.T) {
duration := 100 * time.Millisecond
ref := "foo"
sb := &slowpublish{duration}
cb, _ := NewCaching(sb)
sp := &slowpublish{duration}
cb, _ := NewCaching(sp)
previousDigest := "not-a-digest"
// Each iteration, we test that the first publish is slow and subsequent

95
pkg/publish/tarball.go Normal file
View File

@ -0,0 +1,95 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package publish
import (
"fmt"
"log"
"strings"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/tarball"
)
type TarballPublisher struct {
file string
base string
namer Namer
tags []string
refs map[name.Reference]v1.Image
}
// NewTarball returns a new publish.Interface that saves images to a tarball.
func NewTarball(file, base string, namer Namer, tags []string) *TarballPublisher {
return &TarballPublisher{
file: file,
base: base,
namer: namer,
tags: tags,
refs: make(map[name.Reference]v1.Image),
}
}
// Publish implements publish.Interface.
func (t *TarballPublisher) Publish(img v1.Image, s string) (name.Reference, error) {
// https://github.com/google/go-containerregistry/issues/212
s = strings.ToLower(s)
for _, tagName := range t.tags {
tag, err := name.ParseReference(fmt.Sprintf("%s/%s:%s", t.base, t.namer(s), tagName))
if err != nil {
return nil, err
}
t.refs[tag] = img
}
h, err := img.Digest()
if err != nil {
return nil, err
}
if len(t.tags) == 0 {
ref, err := name.ParseReference(fmt.Sprintf("%s/%s@%s", t.base, t.namer(s), h))
if err != nil {
return nil, err
}
t.refs[ref] = img
}
ref := fmt.Sprintf("%s/%s@%s", t.base, t.namer(s), h)
if len(t.tags) == 1 && t.tags[0] != defaultTags[0] {
// If a single tag is explicitly set (not latest), then this
// is probably a release, so include the tag in the reference.
ref = fmt.Sprintf("%s/%s:%s@%s", t.base, t.namer(s), t.tags[0], h)
}
dig, err := name.NewDigest(ref)
if err != nil {
return nil, err
}
return &dig, nil
}
func (t *TarballPublisher) Close() error {
log.Printf("Saving %v", t.file)
if err := tarball.MultiRefWriteToFile(t.file, t.refs); err != nil {
// Bad practice, but we log this here because right now we just defer the Close.
log.Printf("failed to save %q: %v", t.file, err)
return err
}
log.Printf("Saved %v", t.file)
return nil
}

View File

@ -0,0 +1,69 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package publish
import (
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/random"
)
func TestTarball(t *testing.T) {
img, err := random.Image(1024, 1)
if err != nil {
t.Fatalf("random.Image() = %v", err)
}
base := "blah"
importpath := "github.com/Google/go-containerregistry/cmd/crane"
fp, err := ioutil.TempFile("", "")
if err != nil {
t.Fatal(err)
}
defer fp.Close()
defer os.Remove(fp.Name())
expectedRepo := fmt.Sprintf("%s/%s", base, md5Hash(strings.ToLower(importpath)))
tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", "example.com", expectedRepo))
if err != nil {
t.Fatalf("NewTag() = %v", err)
}
repoName := fmt.Sprintf("%s/%s", "example.com", base)
tagss := [][]string{{
// no tags
}, {
// one tag
"v0.1.0",
}, {
// multiple tags
"latest",
"debug",
}}
for _, tags := range tagss {
tp := NewTarball(fp.Name(), repoName, md5Hash, tags)
if d, err := tp.Publish(img, importpath); err != nil {
t.Errorf("Publish() = %v", err)
} else if !strings.HasPrefix(d.String(), tag.Repository.String()) {
t.Errorf("Publish() = %v, wanted prefix %v", d, tag.Repository)
}
}
}

View File

@ -12,22 +12,27 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package options
package layout
import (
"github.com/spf13/cobra"
"io"
"io/ioutil"
"os"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// LocalOptions represents options for the ko binary.
type LocalOptions struct {
// Local publishes images to a local docker daemon.
Local bool
InsecureRegistry bool
// Blob returns a blob with the given hash from the Path.
func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) {
return os.Open(l.blobPath(h))
}
func AddLocalArg(cmd *cobra.Command, lo *LocalOptions) {
cmd.Flags().BoolVarP(&lo.Local, "local", "L", lo.Local,
"Whether to publish images to a local docker daemon vs. a registry.")
cmd.Flags().BoolVar(&lo.InsecureRegistry, "insecure-registry", lo.InsecureRegistry,
"Whether to skip TLS verification on the registry")
// Bytes is a convenience function to return a blob from the Path as
// a byte slice.
func (l Path) Bytes(h v1.Hash) ([]byte, error) {
return ioutil.ReadFile(l.blobPath(h))
}
func (l Path) blobPath(h v1.Hash) string {
return l.path("blobs", h.Algorithm, h.Hex)
}

View File

@ -12,18 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package options
import (
"github.com/spf13/cobra"
)
// TagsOptions holds the list of tags to tag the built image
type TagsOptions struct {
Tags []string
}
func AddTagsArg(cmd *cobra.Command, ta *TagsOptions) {
cmd.Flags().StringSliceVarP(&ta.Tags, "tags", "t", []string{"latest"},
"Which tags to use for the produced image instead of the default 'latest' tag.")
}
// Package layout provides facilities for reading/writing artifacts from/to
// an OCI image layout on disk, see:
//
// https://github.com/opencontainers/image-spec/blob/master/image-layout.md
package layout

View File

@ -0,0 +1,131 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import (
"fmt"
"io"
"sync"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
)
type layoutImage struct {
path Path
desc v1.Descriptor
manifestLock sync.Mutex // Protects rawManifest
rawManifest []byte
}
var _ partial.CompressedImageCore = (*layoutImage)(nil)
// Image reads a v1.Image with digest h from the Path.
func (l Path) Image(h v1.Hash) (v1.Image, error) {
ii, err := l.ImageIndex()
if err != nil {
return nil, err
}
return ii.Image(h)
}
func (li *layoutImage) MediaType() (types.MediaType, error) {
return li.desc.MediaType, nil
}
// Implements WithManifest for partial.Blobset.
func (li *layoutImage) Manifest() (*v1.Manifest, error) {
return partial.Manifest(li)
}
func (li *layoutImage) RawManifest() ([]byte, error) {
li.manifestLock.Lock()
defer li.manifestLock.Unlock()
if li.rawManifest != nil {
return li.rawManifest, nil
}
b, err := li.path.Bytes(li.desc.Digest)
if err != nil {
return nil, err
}
li.rawManifest = b
return li.rawManifest, nil
}
func (li *layoutImage) RawConfigFile() ([]byte, error) {
manifest, err := li.Manifest()
if err != nil {
return nil, err
}
return li.path.Bytes(manifest.Config.Digest)
}
func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) {
manifest, err := li.Manifest()
if err != nil {
return nil, err
}
if h == manifest.Config.Digest {
return partial.CompressedLayer(&compressedBlob{
path: li.path,
desc: manifest.Config,
}), nil
}
for _, desc := range manifest.Layers {
if h == desc.Digest {
switch desc.MediaType {
case types.OCILayer, types.DockerLayer:
return partial.CompressedToLayer(&compressedBlob{
path: li.path,
desc: desc,
})
default:
// TODO: We assume everything is a compressed blob, but that might not be true.
// TODO: Handle foreign layers.
return nil, fmt.Errorf("unexpected media type: %v for layer: %v", desc.MediaType, desc.Digest)
}
}
}
return nil, fmt.Errorf("could not find layer in image: %s", h)
}
type compressedBlob struct {
path Path
desc v1.Descriptor
}
func (b *compressedBlob) Digest() (v1.Hash, error) {
return b.desc.Digest, nil
}
func (b *compressedBlob) Compressed() (io.ReadCloser, error) {
return b.path.Blob(b.desc.Digest)
}
func (b *compressedBlob) Size() (int64, error) {
return b.desc.Size, nil
}
func (b *compressedBlob) MediaType() (types.MediaType, error) {
return b.desc.MediaType, nil
}

View File

@ -0,0 +1,153 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
)
var _ v1.ImageIndex = (*layoutIndex)(nil)
type layoutIndex struct {
mediaType types.MediaType
path Path
rawIndex []byte
}
// ImageIndexFromPath is a convenience function which constructs a Path and returns its v1.ImageIndex.
func ImageIndexFromPath(path string) (v1.ImageIndex, error) {
lp, err := FromPath(path)
if err != nil {
return nil, err
}
return lp.ImageIndex()
}
// ImageIndex returns a v1.ImageIndex for the Path.
func (l Path) ImageIndex() (v1.ImageIndex, error) {
rawIndex, err := ioutil.ReadFile(l.path("index.json"))
if err != nil {
return nil, err
}
idx := &layoutIndex{
mediaType: types.OCIImageIndex,
path: l,
rawIndex: rawIndex,
}
return idx, nil
}
func (i *layoutIndex) MediaType() (types.MediaType, error) {
return i.mediaType, nil
}
func (i *layoutIndex) Digest() (v1.Hash, error) {
return partial.Digest(i)
}
func (i *layoutIndex) Size() (int64, error) {
return partial.Size(i)
}
func (i *layoutIndex) IndexManifest() (*v1.IndexManifest, error) {
var index v1.IndexManifest
err := json.Unmarshal(i.rawIndex, &index)
return &index, err
}
func (i *layoutIndex) RawManifest() ([]byte, error) {
return i.rawIndex, nil
}
func (i *layoutIndex) Image(h v1.Hash) (v1.Image, error) {
// Look up the digest in our manifest first to return a better error.
desc, err := i.findDescriptor(h)
if err != nil {
return nil, err
}
if !isExpectedMediaType(desc.MediaType, types.OCIManifestSchema1, types.DockerManifestSchema2) {
return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType)
}
img := &layoutImage{
path: i.path,
desc: *desc,
}
return partial.CompressedToImage(img)
}
func (i *layoutIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) {
// Look up the digest in our manifest first to return a better error.
desc, err := i.findDescriptor(h)
if err != nil {
return nil, err
}
if !isExpectedMediaType(desc.MediaType, types.OCIImageIndex, types.DockerManifestList) {
return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType)
}
rawIndex, err := i.path.Bytes(h)
if err != nil {
return nil, err
}
return &layoutIndex{
mediaType: desc.MediaType,
path: i.path,
rawIndex: rawIndex,
}, nil
}
func (i *layoutIndex) Blob(h v1.Hash) (io.ReadCloser, error) {
return i.path.Blob(h)
}
func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) {
im, err := i.IndexManifest()
if err != nil {
return nil, err
}
for _, desc := range im.Manifests {
if desc.Digest == h {
return &desc, nil
}
}
return nil, fmt.Errorf("could not find descriptor in index: %s", h)
}
// TODO: Pull this out into methods on types.MediaType? e.g. instead, have:
// * mt.IsIndex()
// * mt.IsImage()
func isExpectedMediaType(mt types.MediaType, expected ...types.MediaType) bool {
for _, allowed := range expected {
if mt == allowed {
return true
}
}
return false
}

View File

@ -0,0 +1,25 @@
// Copyright 2019 The original author or authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import "path/filepath"
// Path represents an OCI image layout rooted in a file system path
type Path string
func (l Path) path(elem ...string) string {
complete := []string{string(l)}
return filepath.Join(append(complete, elem...)...)
}

View File

@ -0,0 +1,42 @@
package layout
import v1 "github.com/google/go-containerregistry/pkg/v1"
// Option is a functional option for Layout.
//
// TODO: We'll need to change this signature to support Sparse/Thin images.
// Or, alternatively, wrap it in a sparse.Image that returns an empty list for layers?
type Option func(*v1.Descriptor) error
// WithAnnotations adds annotations to the artifact descriptor.
func WithAnnotations(annotations map[string]string) Option {
return func(desc *v1.Descriptor) error {
if desc.Annotations == nil {
desc.Annotations = make(map[string]string)
}
for k, v := range annotations {
desc.Annotations[k] = v
}
return nil
}
}
// WithURLs adds urls to the artifact descriptor.
func WithURLs(urls []string) Option {
return func(desc *v1.Descriptor) error {
if desc.URLs == nil {
desc.URLs = []string{}
}
desc.URLs = append(desc.URLs, urls...)
return nil
}
}
// WithPlatform sets the platform of the artifact descriptor.
func WithPlatform(platform v1.Platform) Option {
return func(desc *v1.Descriptor) error {
desc.Platform = &platform
return nil
}
}

View File

@ -1,4 +1,4 @@
// Copyright 2018 Google LLC All Rights Reserved.
// Copyright 2019 The original author or authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,19 +12,21 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package options
package layout
import (
"github.com/spf13/cobra"
"os"
"path/filepath"
)
// PublishOptions represents options for the ko binary.
type PublishOptions struct {
// Path is the import path of the binary to publish.
Path string
}
// FromPath reads an OCI image layout at path and constructs a layout.Path.
func FromPath(path string) (Path, error) {
// TODO: check oci-layout exists
func AddImageArg(cmd *cobra.Command, lo *PublishOptions) {
cmd.Flags().StringVarP(&lo.Path, "image", "i", lo.Path,
"The import path of the binary to publish.")
_, err := os.Stat(filepath.Join(path, "index.json"))
if err != nil {
return "", err
}
return Path(path), nil
}

View File

@ -0,0 +1,301 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"os"
"path/filepath"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"golang.org/x/sync/errgroup"
)
var layoutFile = `{
"imageLayoutVersion": "1.0.0"
}`
// AppendImage writes a v1.Image to the Path and updates
// the index.json to reference it.
func (l Path) AppendImage(img v1.Image, options ...Option) error {
if err := l.writeImage(img); err != nil {
return err
}
mt, err := img.MediaType()
if err != nil {
return err
}
d, err := img.Digest()
if err != nil {
return err
}
manifest, err := img.RawManifest()
if err != nil {
return err
}
desc := v1.Descriptor{
MediaType: mt,
Size: int64(len(manifest)),
Digest: d,
}
for _, opt := range options {
if err := opt(&desc); err != nil {
return err
}
}
return l.AppendDescriptor(desc)
}
// AppendIndex writes a v1.ImageIndex to the Path and updates
// the index.json to reference it.
func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error {
if err := l.writeIndex(ii); err != nil {
return err
}
mt, err := ii.MediaType()
if err != nil {
return err
}
d, err := ii.Digest()
if err != nil {
return err
}
manifest, err := ii.RawManifest()
if err != nil {
return err
}
desc := v1.Descriptor{
MediaType: mt,
Size: int64(len(manifest)),
Digest: d,
}
for _, opt := range options {
if err := opt(&desc); err != nil {
return err
}
}
return l.AppendDescriptor(desc)
}
// AppendDescriptor adds a descriptor to the index.json of the Path.
func (l Path) AppendDescriptor(desc v1.Descriptor) error {
ii, err := l.ImageIndex()
if err != nil {
return err
}
index, err := ii.IndexManifest()
if err != nil {
return err
}
index.Manifests = append(index.Manifests, desc)
rawIndex, err := json.MarshalIndent(index, "", " ")
if err != nil {
return err
}
return l.writeFile("index.json", rawIndex)
}
func (l Path) writeFile(name string, data []byte) error {
if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) {
return err
}
return ioutil.WriteFile(l.path(name), data, os.ModePerm)
}
// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at
// blobs/{hash.Algorithm}/{hash.Hex}.
func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error {
dir := l.path("blobs", hash.Algorithm)
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
return err
}
file := filepath.Join(dir, hash.Hex)
if _, err := os.Stat(file); err == nil {
// Blob already exists, that's fine.
return nil
}
w, err := os.Create(file)
if err != nil {
return err
}
defer w.Close()
_, err = io.Copy(w, r)
return err
}
// TODO: A streaming version of WriteBlob so we don't have to know the hash
// before we write it.
// TODO: For streaming layers we should write to a tmp file then Rename to the
// final digest.
func (l Path) writeLayer(layer v1.Layer) error {
d, err := layer.Digest()
if err != nil {
return err
}
r, err := layer.Compressed()
if err != nil {
return err
}
return l.WriteBlob(d, r)
}
func (l Path) writeImage(img v1.Image) error {
layers, err := img.Layers()
if err != nil {
return err
}
// Write the layers concurrently.
var g errgroup.Group
for _, layer := range layers {
layer := layer
g.Go(func() error {
return l.writeLayer(layer)
})
}
if err := g.Wait(); err != nil {
return err
}
// Write the config.
cfgName, err := img.ConfigName()
if err != nil {
return err
}
cfgBlob, err := img.RawConfigFile()
if err != nil {
return err
}
if err := l.WriteBlob(cfgName, ioutil.NopCloser(bytes.NewReader(cfgBlob))); err != nil {
return err
}
// Write the img manifest.
d, err := img.Digest()
if err != nil {
return err
}
manifest, err := img.RawManifest()
if err != nil {
return err
}
return l.WriteBlob(d, ioutil.NopCloser(bytes.NewReader(manifest)))
}
func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error {
index, err := ii.IndexManifest()
if err != nil {
return err
}
// Walk the descriptors and write any v1.Image or v1.ImageIndex that we find.
// If we come across something we don't expect, just write it as a blob.
for _, desc := range index.Manifests {
switch desc.MediaType {
case types.OCIImageIndex, types.DockerManifestList:
ii, err := ii.ImageIndex(desc.Digest)
if err != nil {
return err
}
if err := l.writeIndex(ii); err != nil {
return err
}
case types.OCIManifestSchema1, types.DockerManifestSchema2:
img, err := ii.Image(desc.Digest)
if err != nil {
return err
}
if err := l.writeImage(img); err != nil {
return err
}
default:
// TODO: The layout could reference arbitrary things, which we should
// probably just pass through.
}
}
rawIndex, err := ii.RawManifest()
if err != nil {
return err
}
return l.writeFile(indexFile, rawIndex)
}
func (l Path) writeIndex(ii v1.ImageIndex) error {
// Always just write oci-layout file, since it's small.
if err := l.writeFile("oci-layout", []byte(layoutFile)); err != nil {
return err
}
h, err := ii.Digest()
if err != nil {
return err
}
indexFile := filepath.Join("blobs", h.Algorithm, h.Hex)
return l.writeIndexToFile(indexFile, ii)
}
// Write constructs a Path at path from an ImageIndex.
//
// The contents are written in the following format:
// At the top level, there is:
// One oci-layout file containing the version of this image-layout.
// One index.json file listing descriptors for the contained images.
// Under blobs/, there is, for each image:
// One file for each layer, named after the layer's SHA.
// One file for each config blob, named after its SHA.
// One file for each manifest blob, named after its SHA.
func Write(path string, ii v1.ImageIndex) (Path, error) {
lp := Path(path)
// Always just write oci-layout file, since it's small.
if err := lp.writeFile("oci-layout", []byte(layoutFile)); err != nil {
return "", err
}
// TODO create blobs/ in case there is a blobs file which would prevent the directory from being created
return lp, lp.writeIndexToFile("index.json", ii)
}

View File

@ -8,5 +8,3 @@ require (
github.com/stretchr/testify v1.2.2
golang.org/x/sys v0.0.0-20190422165155-953cdadca894
)
go 1.13

1
vendor/modules.txt vendored
View File

@ -93,6 +93,7 @@ github.com/google/go-containerregistry/pkg/name
github.com/google/go-containerregistry/pkg/v1
github.com/google/go-containerregistry/pkg/v1/daemon
github.com/google/go-containerregistry/pkg/v1/empty
github.com/google/go-containerregistry/pkg/v1/layout
github.com/google/go-containerregistry/pkg/v1/mutate
github.com/google/go-containerregistry/pkg/v1/partial
github.com/google/go-containerregistry/pkg/v1/random