// Copyright 2018 Google LLC All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package build import ( "archive/tar" "bytes" "compress/gzip" "encoding/json" "errors" gb "go/build" "io" "io/ioutil" "log" "os" "os/exec" "path/filepath" "strings" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/tarball" ) const ( appDir = "/ko-app" defaultAppFilename = "ko-app" ) // GetBase takes an importpath and returns a base v1.Image. type GetBase func(string) (v1.Image, error) type builder func(string, v1.Platform, bool) (string, error) type gobuild struct { getBase GetBase creationTime v1.Time build builder disableOptimizations bool mod *modInfo } // Option is a functional option for NewGo. type Option func(*gobuildOpener) error type gobuildOpener struct { getBase GetBase creationTime v1.Time build builder disableOptimizations bool mod *modInfo } func (gbo *gobuildOpener) Open() (Interface, error) { if gbo.getBase == nil { return nil, errors.New("a way of providing base images must be specified, see build.WithBaseImages") } return &gobuild{ getBase: gbo.getBase, creationTime: gbo.creationTime, build: gbo.build, disableOptimizations: gbo.disableOptimizations, mod: gbo.mod, }, nil } // https://golang.org/pkg/cmd/go/internal/modinfo/#ModulePublic type modInfo struct { Path string Dir string } // moduleInfo returns the module path and module root directory for a project // using go modules, otherwise returns nil. // // Related: https://github.com/golang/go/issues/26504 func moduleInfo() *modInfo { output, err := exec.Command("go", "list", "-mod=readonly", "-m", "-json").Output() if err != nil { return nil } var info modInfo if err := json.Unmarshal(output, &info); err != nil { return nil } return &info } // NewGo returns a build.Interface implementation that: // 1. builds go binaries named by importpath, // 2. containerizes the binary on a suitable base, func NewGo(options ...Option) (Interface, error) { gbo := &gobuildOpener{ build: build, mod: moduleInfo(), } for _, option := range options { if err := option(gbo); err != nil { return nil, err } } return gbo.Open() } // IsSupportedReference implements build.Interface // // Only valid importpaths that provide commands (i.e., are "package main") are // supported. func (g *gobuild) IsSupportedReference(s string) bool { p, err := g.importPackage(s) if err != nil { return false } return p.IsCommand() } var moduleErr = errors.New("unmatched importPackage with gomodules") // importPackage wraps go/build.Import to handle go modules. // // Note that we will fall back to GOPATH if the project isn't using go modules. func (g *gobuild) importPackage(s string) (*gb.Package, error) { if g.mod == nil { return gb.Import(s, gb.Default.GOPATH, gb.ImportComment) } // If we're inside a go modules project, try to use the module's directory // as our source root to import: // * paths that match module path prefix (they should be in this project) // * relative paths (they should also be in this project) if strings.HasPrefix(s, g.mod.Path) || gb.IsLocalImport(s) { return gb.Import(s, g.mod.Dir, gb.ImportComment) } return nil, moduleErr } func build(ip string, platform v1.Platform, disableOptimizations bool) (string, error) { tmpDir, err := ioutil.TempDir("", "ko") if err != nil { return "", err } file := filepath.Join(tmpDir, "out") args := make([]string, 0, 6) args = append(args, "build") if disableOptimizations { // Disable optimizations (-N) and inlining (-l). args = append(args, "-gcflags", "all=-N -l") } args = append(args, "-o", file) args = append(args, ip) cmd := exec.Command("go", args...) // Last one wins defaultEnv := []string{ "CGO_ENABLED=0", "GOOS=" + platform.OS, "GOARCH=" + platform.Architecture, } cmd.Env = append(defaultEnv, os.Environ()...) var output bytes.Buffer cmd.Stderr = &output cmd.Stdout = &output log.Printf("Building %s", ip) if err := cmd.Run(); err != nil { os.RemoveAll(tmpDir) log.Printf("Unexpected error running \"go build\": %v\n%v", err, output.String()) return "", err } return file, nil } func appFilename(importpath string) string { base := filepath.Base(importpath) // If we fail to determine a good name from the importpath then use a // safe default. if base == "." || base == string(filepath.Separator) { return defaultAppFilename } return base } func tarAddDirectories(tw *tar.Writer, dir string) error { if dir == "." || dir == string(filepath.Separator) { return nil } // Write parent directories first if err := tarAddDirectories(tw, filepath.Dir(dir)); err != nil { return err } // write the directory header to the tarball archive if err := tw.WriteHeader(&tar.Header{ Name: dir, Typeflag: tar.TypeDir, // Use a fixed Mode, so that this isn't sensitive to the directory and umask // under which it was created. Additionally, windows can only set 0222, // 0444, or 0666, none of which are executable. Mode: 0555, }); err != nil { return err } return nil } func tarBinary(name, binary string) (*bytes.Buffer, error) { buf := bytes.NewBuffer(nil) // Compress this before calling tarball.LayerFromOpener, since it eagerly // calculates digests and diffids. This prevents us from double compressing // the layer when we have to actually upload the blob. // // https://github.com/google/go-containerregistry/issues/413 gw, _ := gzip.NewWriterLevel(buf, gzip.BestSpeed) defer gw.Close() tw := tar.NewWriter(gw) defer tw.Close() // write the parent directories to the tarball archive if err := tarAddDirectories(tw, filepath.Dir(name)); err != nil { return nil, err } file, err := os.Open(binary) if err != nil { return nil, err } defer file.Close() stat, err := file.Stat() if err != nil { return nil, err } header := &tar.Header{ Name: name, Size: stat.Size(), Typeflag: tar.TypeReg, // Use a fixed Mode, so that this isn't sensitive to the directory and umask // under which it was created. Additionally, windows can only set 0222, // 0444, or 0666, none of which are executable. Mode: 0555, } // write the header to the tarball archive if err := tw.WriteHeader(header); err != nil { return nil, err } // copy the file data to the tarball if _, err := io.Copy(tw, file); err != nil { return nil, err } return buf, nil } func (g *gobuild) kodataPath(s string) (string, error) { p, err := g.importPackage(s) if err != nil { return "", err } return filepath.Join(p.Dir, "kodata"), nil } // Where kodata lives in the image. const kodataRoot = "/var/run/ko" // walkRecursive performs a filepath.Walk of the given root directory adding it // to the provided tar.Writer with root -> chroot. All symlinks are dereferenced, // which is what leads to recursion when we encounter a directory symlink. func walkRecursive(tw *tar.Writer, root, chroot string) error { return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if path == root { // Add an entry for the root directory of our walk. return tw.WriteHeader(&tar.Header{ Name: chroot, Typeflag: tar.TypeDir, // Use a fixed Mode, so that this isn't sensitive to the directory and umask // under which it was created. Additionally, windows can only set 0222, // 0444, or 0666, none of which are executable. Mode: 0555, }) } if err != nil { return err } // Skip other directories. if info.Mode().IsDir() { return nil } newPath := filepath.Join(chroot, path[len(root):]) path, err = filepath.EvalSymlinks(path) if err != nil { return err } // Chase symlinks. info, err = os.Stat(path) if err != nil { return err } // Skip other directories. if info.Mode().IsDir() { return walkRecursive(tw, path, newPath) } // Open the file to copy it into the tarball. file, err := os.Open(path) if err != nil { return err } defer file.Close() // Copy the file into the image tarball. if err := tw.WriteHeader(&tar.Header{ Name: newPath, Size: info.Size(), Typeflag: tar.TypeReg, // Use a fixed Mode, so that this isn't sensitive to the directory and umask // under which it was created. Additionally, windows can only set 0222, // 0444, or 0666, none of which are executable. Mode: 0555, }); err != nil { return err } _, err = io.Copy(tw, file) return err }) } func (g *gobuild) tarKoData(importpath string) (*bytes.Buffer, error) { buf := bytes.NewBuffer(nil) // Compress this before calling tarball.LayerFromOpener, since it eagerly // calculates digests and diffids. This prevents us from double compressing // the layer when we have to actually upload the blob. // // https://github.com/google/go-containerregistry/issues/413 gw, _ := gzip.NewWriterLevel(buf, gzip.BestSpeed) defer gw.Close() tw := tar.NewWriter(gw) defer tw.Close() root, err := g.kodataPath(importpath) if err != nil { return nil, err } return buf, walkRecursive(tw, root, kodataRoot) } // Build implements build.Interface func (gb *gobuild) Build(s string) (v1.Image, error) { // Determine the appropriate base image for this import path. base, err := gb.getBase(s) if err != nil { return nil, err } cf, err := base.ConfigFile() if err != nil { return nil, err } platform := v1.Platform{ OS: cf.OS, Architecture: cf.Architecture, } // Do the build into a temporary file. file, err := gb.build(s, platform, gb.disableOptimizations) if err != nil { return nil, err } defer os.RemoveAll(filepath.Dir(file)) var layers []mutate.Addendum // Create a layer from the kodata directory under this import path. dataLayerBuf, err := gb.tarKoData(s) if err != nil { return nil, err } dataLayerBytes := dataLayerBuf.Bytes() dataLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBuffer(dataLayerBytes)), nil }) if err != nil { return nil, err } layers = append(layers, mutate.Addendum{ Layer: dataLayer, History: v1.History{ Author: "ko", CreatedBy: "ko publish " + s, Comment: "kodata contents, at $KO_DATA_PATH", }, }) appPath := filepath.Join(appDir, appFilename(s)) // Construct a tarball with the binary and produce a layer. binaryLayerBuf, err := tarBinary(appPath, file) if err != nil { return nil, err } binaryLayerBytes := binaryLayerBuf.Bytes() binaryLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBuffer(binaryLayerBytes)), nil }) if err != nil { return nil, err } layers = append(layers, mutate.Addendum{ Layer: binaryLayer, History: v1.History{ Author: "ko", CreatedBy: "ko publish " + s, Comment: "go build output, at " + appPath, }, }) // Augment the base image with our application layer. withApp, err := mutate.Append(base, layers...) if err != nil { return nil, err } // Start from a copy of the base image's config file, and set // the entrypoint to our app. cfg, err := withApp.ConfigFile() if err != nil { return nil, err } cfg = cfg.DeepCopy() cfg.Config.Entrypoint = []string{appPath} cfg.Config.Env = append(cfg.Config.Env, "KO_DATA_PATH="+kodataRoot) cfg.ContainerConfig = cfg.Config cfg.Author = "github.com/google/ko" image, err := mutate.ConfigFile(withApp, cfg) if err != nil { return nil, err } empty := v1.Time{} if gb.creationTime != empty { return mutate.CreatedAt(image, gb.creationTime) } return image, nil }