[modules] Dependent command packages can now be built with ko (#154)

Thus if you have a (in)direct command package as a dependency
say `myhost.com/go/package/cmd/run` you can now publish this
with the following ko command

  ko publish myhost.com/go/package/cmd/run

pkg/build/gobuild.go

@@ -46,12 +46,17 @@ const (
 type GetBase func(string) (v1.Image, error)
 type builder func(context.Context, string, v1.Platform, bool) (string, error)
 
+type buildContext interface {
+	Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error)
+}
+
 type gobuild struct {
 	getBase              GetBase
 	creationTime         v1.Time
 	build                builder
 	disableOptimizations bool
-	mod                  *modInfo
+	mod                  *modules
+	buildContext         buildContext
 }
 
 // Option is a functional option for NewGo.
@@ -62,7 +67,8 @@ type gobuildOpener struct {
 	creationTime         v1.Time
 	build                builder
 	disableOptimizations bool
-	mod                  *modInfo
+	mod                  *modules
+	buildContext         buildContext
 }
 
 func (gbo *gobuildOpener) Open() (Interface, error) {
@@ -75,38 +81,80 @@ func (gbo *gobuildOpener) Open() (Interface, error) {
 		build:                gbo.build,
 		disableOptimizations: gbo.disableOptimizations,
 		mod:                  gbo.mod,
+		buildContext:         gbo.buildContext,
 	}, nil
 }
 
 // https://golang.org/pkg/cmd/go/internal/modinfo/#ModulePublic
+type modules struct {
+	main *modInfo
+	deps map[string]*modInfo
+}
+
 type modInfo struct {
 	Path string
 	Dir  string
+	Main bool
 }
 
 // moduleInfo returns the module path and module root directory for a project
 // using go modules, otherwise returns nil.
 //
 // Related: https://github.com/golang/go/issues/26504
-func moduleInfo() *modInfo {
+func moduleInfo() (*modules, error) {
-	output, err := exec.Command("go", "list", "-mod=readonly", "-m", "-json").Output()
+	modules := modules{
+		deps: make(map[string]*modInfo),
+	}
+
+	// TODO we read all the output as a single byte array - it may
+	// be possible & more efficient to stream it
+	output, err := exec.Command("go", "list", "-mod=readonly", "-json", "-m", "all").Output()
 	if err != nil {
-		return nil
+		return nil, nil
 	}
-	var info modInfo
+
-	if err := json.Unmarshal(output, &info); err != nil {
+	dec := json.NewDecoder(bytes.NewReader(output))
-		return nil
+
+	for {
+		var info modInfo
+
+		err := dec.Decode(&info)
+		if err == io.EOF {
+			// all done
+			break
+		}
+
+		modules.deps[info.Path] = &info
+
+		if info.Main {
+			modules.main = &info
+		}
+
+		if err != nil {
+			return nil, fmt.Errorf("error reading module data %w", err)
+		}
 	}
-	return &info
+
+	if modules.main == nil {
+		return nil, fmt.Errorf("couldn't find main module")
+	}
+
+	return &modules, nil
 }
 
 // NewGo returns a build.Interface implementation that:
 //  1. builds go binaries named by importpath,
 //  2. containerizes the binary on a suitable base,
 func NewGo(options ...Option) (Interface, error) {
+	module, err := moduleInfo()
+	if err != nil {
+		return nil, err
+	}
+
 	gbo := &gobuildOpener{
-		build: build,
+		build:        build,
-		mod:   moduleInfo(),
+		mod:          module,
+		buildContext: &gb.Default,
 	}
 
 	for _, option := range options {
@@ -141,7 +189,7 @@ func (g *gobuild) IsSupportedReference(s string) bool {
 // Note that we will fall back to GOPATH if the project isn't using go modules.
 func (g *gobuild) importPackage(ref reference) (*gb.Package, error) {
 	if g.mod == nil {
-		return gb.Import(ref.Path(), gb.Default.GOPATH, gb.ImportComment)
+		return g.buildContext.Import(ref.Path(), gb.Default.GOPATH, gb.ImportComment)
 	}
 
 	// If we're inside a go modules project, try to use the module's directory
@@ -149,8 +197,11 @@ func (g *gobuild) importPackage(ref reference) (*gb.Package, error) {
 	// * any strict reference we get
 	// * paths that match module path prefix (they should be in this project)
 	// * relative paths (they should also be in this project)
-	if ref.IsStrict() || strings.HasPrefix(ref.Path(), g.mod.Path) || gb.IsLocalImport(ref.Path()) {
+	// * path is a module
-		return gb.Import(ref.Path(), g.mod.Dir, gb.ImportComment)
+
+	_, isDep := g.mod.deps[ref.Path()]
+	if ref.IsStrict() || strings.HasPrefix(ref.Path(), g.mod.main.Path) || gb.IsLocalImport(ref.Path()) || isDep {
+		return g.buildContext.Import(ref.Path(), g.mod.main.Dir, gb.ImportComment)
 	}
 
 	return nil, fmt.Errorf("unmatched importPackage %q with gomodules", ref.String())

pkg/build/gobuild_test.go

@@ -17,6 +17,7 @@ package build
 import (
 	"archive/tar"
 	"context"
+	gb "go/build"
 	"io"
 	"io/ioutil"
 	"path"
@@ -69,12 +70,31 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) {
 	if err != nil {
 		t.Fatalf("random.Image() = %v", err)
 	}
-	mod := &modInfo{
+
-		Path: "github.com/google/ko/cmd/ko/test",
+	mods := &modules{
-		Dir:  ".",
+		main: &modInfo{
+			Path: "github.com/google/ko/cmd/ko/test",
+			Dir:  ".",
+		},
+		deps: map[string]*modInfo{
+			"github.com/some/module/cmd": &modInfo{
+				Path: "github.com/some/module/cmd",
+				Dir:  ".",
+			},
+		},
 	}
 
-	ng, err := NewGo(WithBaseImages(func(string) (v1.Image, error) { return base, nil }), withModuleInfo(mod))
+	opts := []Option{
+		WithBaseImages(func(string) (v1.Image, error) { return base, nil }),
+		withModuleInfo(mods),
+		withBuildContext(stubBuildContext{
+			// make all referenced deps commands
+			"github.com/google/ko/cmd/ko/test": &gb.Package{Name: "main"},
+			"github.com/some/module/cmd":       &gb.Package{Name: "main"},
+		}),
+	}
+
+	ng, err := NewGo(opts...)
 	if err != nil {
 		t.Fatalf("NewGo() = %v", err)
 	}
@@ -82,6 +102,7 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) {
 	// Supported import paths.
 	for _, importpath := range []string{
 		"github.com/google/ko/cmd/ko/test", // ko can build the test package.
+		"github.com/some/module/cmd",       // ko can build commands in dependent modules
 	} {
 		t.Run(importpath, func(t *testing.T) {
 			if !ng.IsSupportedReference(importpath) {
@@ -376,3 +397,9 @@ func TestGoBuild(t *testing.T) {
 		}
 	})
 }
+
+type stubBuildContext map[string]*gb.Package
+
+func (s stubBuildContext) Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error) {
+	return s[path], nil
+}

pkg/build/options.go

@@ -57,9 +57,16 @@ func withBuilder(b builder) Option {
 // withModulePath is a functional option for overriding the module path for
 // the current ko invocation.
 // This is exposed for testing.
-func withModuleInfo(mi *modInfo) Option {
+func withModuleInfo(m *modules) Option {
 	return func(gbo *gobuildOpener) error {
-		gbo.mod = mi
+		gbo.mod = m
+		return nil
+	}
+}
+
+func withBuildContext(b buildContext) Option {
+	return func(gbo *gobuildOpener) error {
+		gbo.buildContext = b
 		return nil
 	}
 }