Add support for Go build flags (#340)

There are use cases, where multiple Go build flags need to be set. However, the
environment variable to pass flags to Go build has some limits for `ldFlags`.

Add GoReleaser inspired configuration section to `.ko.yaml` to support setting
specific Go build and ldFlags to be used by the build. Like GoReleaser the
content of the configuration can use Go templates. Currently, only a section
for environment variables is included.

In order to reduce dependency overhead, only the respective config structs from
https://github.com/goreleaser/goreleaser/blob/master/pkg/config/config.go are
used internally to load from `.ko.yaml`.

README.md

@@ -123,6 +123,42 @@ baseImageOverrides:
   github.com/my-user/my-repo/cmd/foo: registry.example.com/base/for/foo
 ```
 
+### Overriding Go build settings
+
+By default, `ko` builds the binary with no additional build flags other than
+`--trimpath` (depending on the Go version). You can replace the default build
+arguments by providing build flags and ldflags using a
+[GoReleaser](https://github.com/goreleaser/goreleaser) influenced `builds`
+configuration section in your `.ko.yaml`.
+
+```yaml
+builds:
+- id: foo
+  main: ./foobar/foo
+  flags:
+  - -tags
+  - netgo
+  ldflags:
+  - -s -w
+  - -extldflags "-static"
+  - -X main.version={{.Env.VERSION}}
+- id: bar
+  main: ./foobar/bar/main.go
+  ldflags:
+  - -s
+  - -w
+```
+
+For the build, `ko` will pick the entry based on the respective import path
+being used. It will be matched against the local path that is configured using
+`dir` and `main`. In the context of `ko`, it is fine just to specify `main`
+with the intended import path.
+
+_Please note:_ Even though the configuration section is similar to the
+[GoReleaser `builds` section](https://goreleaser.com/customization/build/),
+only the `flags` and `ldflags` fields are currently supported. Also, the
+templating support is currently limited to environment variables only.
+
 ## Naming Images
 
 `ko` provides a few different strategies for naming the image it pushes, to
@@ -335,10 +371,17 @@ is a common way to embed version info in go binaries (In fact, we do this for
 this flag directly; however, you can use the `GOFLAGS` environment variable
 instead:
 
-```
+```sh
 GOFLAGS="-ldflags=-X=main.version=1.2.3" ko publish .
 ```
 
+## How can I set multiple `ldflags`?
+
+Currently, there is a limitation that does not allow to set multiple arguments
+in `ldflags` using `GOFLAGS`. Using `-ldflags` multiple times also does not
+work. In this use case, it works best to use the [`builds` section](#overriding-go-build-settings)
+in the `.ko.yaml` file.
+
 ## Why are my images all created in 1970?
 
 In order to support [reproducible builds](https://reproducible-builds.org), `ko`

pkg/build/config.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2021 Google LLC All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package build
+
+import "strings"
+
+// Note: The structs, types, and functions are based upon GoReleaser build
+// configuration to have a loosely compatible YAML configuration:
+// https://github.com/goreleaser/goreleaser/blob/master/pkg/config/config.go
+
+// StringArray is a wrapper for an array of strings.
+type StringArray []string
+
+// UnmarshalYAML is a custom unmarshaler that wraps strings in arrays.
+func (a *StringArray) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var strings []string
+	if err := unmarshal(&strings); err != nil {
+		var str string
+		if err := unmarshal(&str); err != nil {
+			return err
+		}
+		*a = []string{str}
+	} else {
+		*a = strings
+	}
+	return nil
+}
+
+// FlagArray is a wrapper for an array of strings.
+type FlagArray []string
+
+// UnmarshalYAML is a custom unmarshaler that wraps strings in arrays.
+func (a *FlagArray) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var flags []string
+	if err := unmarshal(&flags); err != nil {
+		var flagstr string
+		if err := unmarshal(&flagstr); err != nil {
+			return err
+		}
+		*a = strings.Fields(flagstr)
+	} else {
+		*a = flags
+	}
+	return nil
+}
+
+// Config contains the build configuration section. The name was changed from
+// the original GoReleaser name to match better with the ko naming.
+//
+// TODO: Introduce support for more fields where possible and where it makes
+///      sense for `ko`, for example ModTimestamp, Env, or GoBinary.
+//
+type Config struct {
+	// ID only serves as an identifier internally
+	ID string `yaml:",omitempty"`
+
+	// Dir is the directory out of which the build should be triggered
+	Dir string `yaml:",omitempty"`
+
+	// Main points to the main package, or the source file with the main
+	// function, in which case only the package will be used for the importpath
+	Main string `yaml:",omitempty"`
+
+	// Ldflags and Flags will be used for the Go build command line arguments
+	Ldflags StringArray `yaml:",omitempty"`
+	Flags   FlagArray   `yaml:",omitempty"`
+
+	// Other GoReleaser fields that are not supported or do not make sense
+	// in the context of ko, for reference or for future use:
+	// Goos         []string    `yaml:",omitempty"`
+	// Goarch       []string    `yaml:",omitempty"`
+	// Goarm        []string    `yaml:",omitempty"`
+	// Gomips       []string    `yaml:",omitempty"`
+	// Targets      []string    `yaml:",omitempty"`
+	// Binary       string      `yaml:",omitempty"`
+	// Env          []string    `yaml:",omitempty"`
+	// Lang         string      `yaml:",omitempty"`
+	// Asmflags     StringArray `yaml:",omitempty"`
+	// Gcflags      StringArray `yaml:",omitempty"`
+	// ModTimestamp string      `yaml:"mod_timestamp,omitempty"`
+	// GoBinary     string      `yaml:",omitempty"`
+}

pkg/build/gobuild.go

@@ -33,6 +33,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"text/template"
 
 	"github.com/containerd/stargz-snapshotter/estargz"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -67,7 +68,7 @@ For more information see:
 // GetBase takes an importpath and returns a base image.
 type GetBase func(context.Context, string) (Result, error)
 
-type builder func(context.Context, string, string, v1.Platform, bool) (string, error)
+type builder func(context.Context, string, string, v1.Platform, Config) (string, error)
 
 type buildContext interface {
 	Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error)
@@ -84,6 +85,7 @@ type gobuild struct {
 	kodataCreationTime   v1.Time
 	build                builder
 	disableOptimizations bool
+	buildConfigs         map[string]Config
 	mod                  *modules
 	buildContext         buildContext
 	platformMatcher      *platformMatcher
@@ -100,6 +102,7 @@ type gobuildOpener struct {
 	kodataCreationTime   v1.Time
 	build                builder
 	disableOptimizations bool
+	buildConfigs         map[string]Config
 	mod                  *modules
 	buildContext         buildContext
 	platform             string
@@ -121,6 +124,7 @@ func (gbo *gobuildOpener) Open() (Interface, error) {
 		kodataCreationTime:   gbo.kodataCreationTime,
 		build:                gbo.build,
 		disableOptimizations: gbo.disableOptimizations,
+		buildConfigs:         gbo.buildConfigs,
 		mod:                  gbo.mod,
 		buildContext:         gbo.buildContext,
 		labels:               gbo.labels,
@@ -354,21 +358,22 @@ func platformToString(p v1.Platform) string {
 	return fmt.Sprintf("%s/%s", p.OS, p.Architecture)
 }
 
-func build(ctx context.Context, ip string, dir string, platform v1.Platform, disableOptimizations bool) (string, error) {
+func build(ctx context.Context, ip string, dir string, platform v1.Platform, config Config) (string, error) {
 	tmpDir, err := ioutil.TempDir("", "ko")
 	if err != nil {
 		return "", err
 	}
 	file := filepath.Join(tmpDir, "out")
 
-	args := make([]string, 0, 7)
-	args = append(args, "build")
-	if disableOptimizations {
-		// Disable optimizations (-N) and inlining (-l).
-		args = append(args, "-gcflags", "all=-N -l")
+	buildArgs, err := createBuildArgs(config)
+	if err != nil {
+		return "", err
 	}
+
+	args := make([]string, 0, 4+len(buildArgs))
+	args = append(args, "build")
+	args = append(args, buildArgs...)
 	args = append(args, "-o", file)
-	args = addGo113TrimPathFlag(args)
 	args = append(args, ip)
 	cmd := exec.CommandContext(ctx, "go", args...)
 	cmd.Dir = dir
@@ -578,6 +583,75 @@ func (g *gobuild) tarKoData(ref reference) (*bytes.Buffer, error) {
 	return buf, walkRecursive(tw, root, kodataRoot, creationTime)
 }
 
+func createTemplateData() map[string]interface{} {
+	envVars := map[string]string{}
+	for _, entry := range os.Environ() {
+		kv := strings.SplitN(entry, "=", 2)
+		envVars[kv[0]] = kv[1]
+	}
+
+	return map[string]interface{}{
+		"Env": envVars,
+	}
+}
+
+func applyTemplating(list []string, data map[string]interface{}) error {
+	for i, entry := range list {
+		tmpl, err := template.New("argsTmpl").Option("missingkey=error").Parse(entry)
+		if err != nil {
+			return err
+		}
+
+		var buf bytes.Buffer
+		if err := tmpl.Execute(&buf, data); err != nil {
+			return err
+		}
+
+		list[i] = buf.String()
+	}
+
+	return nil
+}
+
+func createBuildArgs(buildCfg Config) ([]string, error) {
+	var args []string
+
+	data := createTemplateData()
+
+	if len(buildCfg.Flags) > 0 {
+		if err := applyTemplating(buildCfg.Flags, data); err != nil {
+			return nil, err
+		}
+
+		args = append(args, buildCfg.Flags...)
+	}
+
+	if len(buildCfg.Ldflags) > 0 {
+		if err := applyTemplating(buildCfg.Ldflags, data); err != nil {
+			return nil, err
+		}
+
+		args = append(args, fmt.Sprintf("-ldflags=%s", strings.Join(buildCfg.Ldflags, " ")))
+	}
+
+	return args, nil
+}
+
+func (g *gobuild) configForImportPath(ip string) Config {
+	config, ok := g.buildConfigs[ip]
+	if !ok {
+		// Apply default build flags in case none were supplied
+		config.Flags = addGo113TrimPathFlag(config.Flags)
+	}
+
+	if g.disableOptimizations {
+		// Disable optimizations (-N) and inlining (-l).
+		config.Flags = append(config.Flags, "-gcflags", "all=-N -l")
+	}
+
+	return config
+}
+
 func (g *gobuild) buildOne(ctx context.Context, s string, base v1.Image, platform *v1.Platform) (v1.Image, error) {
 	ref := newRef(s)
 
@@ -594,7 +668,7 @@ func (g *gobuild) buildOne(ctx context.Context, s string, base v1.Image, platfor
 	}
 
 	// Do the build into a temporary file.
-	file, err := g.build(ctx, ref.Path(), g.dir, *platform, g.disableOptimizations)
+	file, err := g.build(ctx, ref.Path(), g.dir, *platform, g.configForImportPath(ref.Path()))
 	if err != nil {
 		return nil, err
 	}

pkg/build/gobuild_test.go

@@ -229,7 +229,7 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) {
 }
 
 // A helper method we use to substitute for the default "build" method.
-func writeTempFile(_ context.Context, s string, _ string, _ v1.Platform, _ bool) (string, error) {
+func writeTempFile(_ context.Context, s string, _ string, _ v1.Platform, _ Config) (string, error) {
 	tmpDir, err := ioutil.TempDir("", "ko")
 	if err != nil {
 		return "", err

pkg/build/options.go

@@ -54,6 +54,18 @@ func WithDisabledOptimizations() Option {
 	}
 }
 
+// WithConfig is a functional option for providing GoReleaser Build influenced
+// build settings for importpaths.
+//
+// Set a fully qualified importpath (e.g. github.com/my-user/my-repo/cmd/app)
+// as the mapping key for the respective Config.
+func WithConfig(buildConfigs map[string]Config) Option {
+	return func(gbo *gobuildOpener) error {
+		gbo.buildConfigs = buildConfigs
+		return nil
+	}
+}
+
 // WithPlatforms is a functional option for building certain platforms for
 // multi-platform base images. To build everything from the base, use "all",
 // otherwise use a comma-separated list of platform specs, i.e.:

pkg/commands/config.go

@@ -22,6 +22,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -37,11 +38,13 @@ import (
 	"github.com/google/ko/pkg/commands/options"
 	"github.com/google/ko/pkg/publish"
 	"github.com/spf13/viper"
+	"golang.org/x/tools/go/packages"
 )
 
 var (
 	defaultBaseImage   string
 	baseImageOverrides map[string]string
+	buildConfigs       map[string]build.Config
 )
 
 // getBaseImage returns a function that determines the base image for a given import path.
@@ -161,6 +164,50 @@ func createCancellableContext() context.Context {
 	return ctx
 }
 
+func createBuildConfigs(baseDir string, configs []build.Config) map[string]build.Config {
+	buildConfigs = make(map[string]build.Config)
+	for i, config := range configs {
+		// Make sure to behave like GoReleaser by defaulting to the current
+		// directory in case the build or main field is not set, check
+		// https://goreleaser.com/customization/build/ for details
+		if config.Dir == "" {
+			config.Dir = "."
+		}
+		if config.Main == "" {
+			config.Main = "."
+		}
+
+		// To behave like GoReleaser, check whether the configured path points to a
+		// source file, and if so, just use the directory it is in
+		var path string
+		if fi, err := os.Stat(filepath.Join(baseDir, config.Dir, config.Main)); err == nil && fi.Mode().IsRegular() {
+			path = filepath.Dir(filepath.Join(config.Dir, config.Main))
+
+		} else {
+			path = filepath.Join(config.Dir, config.Main)
+		}
+
+		// By default, paths configured in the builds section are considered
+		// local import paths, therefore add a "./" equivalent as a prefix to
+		// the constructured import path
+		importPath := fmt.Sprint(".", string(filepath.Separator), path)
+
+		pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName, Dir: baseDir}, importPath)
+		if err != nil {
+			log.Fatalf("'builds': entry #%d does not contain a usuable path (%s): %v", i, importPath, err)
+		}
+
+		if len(pkgs) != 1 {
+			log.Fatalf("'builds': entry #%d results in %d local packages, only 1 is expected", i, len(pkgs))
+		}
+
+		importPath = pkgs[0].PkgPath
+		buildConfigs[importPath] = config
+	}
+
+	return buildConfigs
+}
+
 func init() {
 	// If omitted, use this base image.
 	viper.SetDefault("defaultBaseImage", "gcr.io/distroless/static:nonroot")
@@ -194,4 +241,10 @@ func init() {
 		}
 		baseImageOverrides[k] = v
 	}
+
+	var builds []build.Config
+	if err := viper.UnmarshalKey("builds", &builds); err != nil {
+		log.Fatalf("configuration section 'builds' cannot be parsed")
+	}
+	buildConfigs = createBuildConfigs(".", builds)
 }

pkg/commands/config_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/google/ko/pkg/build"
 	"github.com/google/ko/pkg/commands/options"
 )
 
@@ -45,3 +46,34 @@ func TestOverrideDefaultBaseImageUsingBuildOption(t *testing.T) {
 		t.Errorf("got digest %s, wanted %s", gotDigest, wantDigest)
 	}
 }
+
+func TestCreateBuildConfigs(t *testing.T) {
+	compare := func(expected string, actual string) {
+		if expected != actual {
+			t.Errorf("test case failed: expected '%#v', but actual value is '%#v'", expected, actual)
+		}
+	}
+
+	buildConfigs := []build.Config{
+		{ID: "defaults"},
+		{ID: "OnlyMain", Main: "test"},
+		{ID: "OnlyMainWithFile", Main: "test/main.go"},
+		{ID: "OnlyDir", Dir: "test"},
+		{ID: "DirAndMain", Dir: "test", Main: "main.go"},
+	}
+
+	for _, b := range buildConfigs {
+		for importPath, buildCfg := range createBuildConfigs("../..", []build.Config{b}) {
+			switch buildCfg.ID {
+			case "defaults":
+				compare("github.com/google/ko", importPath)
+
+			case "OnlyMain", "OnlyMainWithFile", "OnlyDir", "DirAndMain":
+				compare("github.com/google/ko/test", importPath)
+
+			default:
+				t.Fatalf("unknown test case: %s", buildCfg.ID)
+			}
+		}
+	}
+}

pkg/commands/resolver.go

@@ -104,6 +104,11 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) {
 		}
 		opts = append(opts, build.WithLabel(parts[0], parts[1]))
 	}
+
+	if len(buildConfigs) > 0 {
+		opts = append(opts, build.WithConfig(buildConfigs))
+	}
+
 	return opts, nil
 }