Build working Windows container images (#374)

* Build working Windows container images

Add e2e tests that run on Windows and cover kodata behavior

* now successfully skipping symlinks on windows :-/

* fix e2e test on windows, that relied on a symlink in kodata after all

* document windows symlink issue

* review feedback

* re-add kodata symlink tests for linux

.github/workflows/e2e.yaml

@@ -0,0 +1,51 @@
+name: Basic e2e test
+
+on:
+  pull_request: 
+    branches: ['main']
+
+jobs:
+  e2e:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+        - ubuntu-latest
+        - windows-latest
+    name: e2e ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.x
+
+    - name: Build and run ko container
+      env:
+        KO_DOCKER_REPO: ko.local
+      shell: bash
+      run: |
+        set -euxo pipefail
+
+        # eval `go env`, compatible with Windows and Linux
+        # cribbed from https://gist.github.com/Syeberman/39d81b1e17d091be5657ecd6fbff0753
+        eval $(go env | sed -r 's/^(set )?(\w+)=("?)(.*)\3$/\2="\4"/gm')
+
+        if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then
+          export KO_DEFAULTBASEIMAGE=mcr.microsoft.com/windows/nanoserver:1809
+        fi
+
+        echo platform is ${GOOS}/${GOARCH}
+        # Build and run the ko binary, which should be runnable.
+        docker run $(go run ./ publish ./ --platform=${GOOS}/${GOARCH} --preserve-import-paths) version
+
+        # Build and run the test/ binary, which should log "Hello there" served from KO_DATA_PATH
+        testimg=$(go run ./ publish ./test --platform=${GOOS}/${GOARCH} --preserve-import-paths)
+        docker run ${testimg} --wait=false 2>&1 | grep "Hello there"
+
+        # Check that symlinks in kodata are chased.
+        # Skip this test on Windows.
+        if [[ "$RUNNER_OS" == "Linux" ]]; then
+          docker run ${testimg} --wait=false -f HEAD
+        fi

.gitignore

@@ -1,3 +1,4 @@
 # Ignore GoLand (IntelliJ) files.
 .idea/
 
+ko

README.md

@@ -408,7 +408,24 @@ timestamp with:
 export KO_DATA_DATE_EPOCH=$(git log -1 --format='%ct')
 ```
 
-## Can I optimize images for [eStargz support](https://github.com/containerd/stargz-snapshotter/blob/v0.2.0/docs/stargz-estargz.md)?
+## Can I build Windows containers?
+
+Yes, but support for Windows containers is new, experimental, and tenuous. Be prepared to file bugs. 🐛
+
+The default base image does not provide a Windows image.
+You can try out building a Windows container image by [setting the base image](#overriding-base-images) to a Windows base image and building with `--platform=windows/amd64` or `--platform=all`:
+
+For example, to build a Windows container image for `ko`, from within this repo:
+
+```
+KO_DEFAULTBASEIMAGE=mcr.microsoft.com/windows/nanoserver:1809 ko publish ./ --platform=windows/amd64
+```
+
+### Known issues 🐛
+
+- Symlinks in `kodata` are ignored when building Windows images; only regular files and directories will be included in the Windows image.
+
+## Can I optimize images for [eStargz support](https://github.com/containerd/stargz-snapshotter/blob/v0.7.0/docs/stargz-estargz.md)?
 
 Yes! Set the environment variable `GGCR_EXPERIMENT_ESTARGZ=1` to produce
 eStargz-optimized images.

pkg/build/gobuild.go

@@ -46,7 +46,6 @@ import (
 )
 
 const (
-	appDir             = "/ko-app"
 	defaultAppFilename = "ko-app"
 
 	gorootWarningTemplate = `NOTICE!
@@ -173,7 +172,6 @@ func moduleInfo(ctx context.Context, dir string) (*modules, error) {
 
 	for {
 		var info modInfo
-
 		err := dec.Decode(&info)
 		if err == io.EOF {
 			// all done
@@ -428,40 +426,42 @@ func appFilename(importpath string) string {
 	return base
 }
 
-func tarAddDirectories(tw *tar.Writer, dir string, creationTime v1.Time) error {
+// userOwnerAndGroupSID is a magic value needed to make the binary executable
-	if dir == "." || dir == string(filepath.Separator) {
+// in a Windows container.
-		return nil
+//
-	}
+// owner: BUILTIN/Users group: BUILTIN/Users ($sddlValue="O:BUG:BU")
+const userOwnerAndGroupSID = "AQAAgBQAAAAkAAAAAAAAAAAAAAABAgAAAAAABSAAAAAhAgAAAQIAAAAAAAUgAAAAIQIAAA=="
 
-	// Write parent directories first
+func tarBinary(name, binary string, creationTime v1.Time, platform *v1.Platform) (*bytes.Buffer, error) {
-	if err := tarAddDirectories(tw, filepath.Dir(dir), creationTime); err != nil {
-		return err
-	}
-
-	// write the directory header to the tarball archive
-	if err := tw.WriteHeader(&tar.Header{
-		Name:     dir,
-		Typeflag: tar.TypeDir,
-		// Use a fixed Mode, so that this isn't sensitive to the directory and umask
-		// under which it was created. Additionally, windows can only set 0222,
-		// 0444, or 0666, none of which are executable.
-		Mode:    0555,
-		ModTime: creationTime.Time,
-	}); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func tarBinary(name, binary string, creationTime v1.Time) (*bytes.Buffer, error) {
 	buf := bytes.NewBuffer(nil)
 	tw := tar.NewWriter(buf)
 	defer tw.Close()
 
-	// write the parent directories to the tarball archive
+	// Write the parent directories to the tarball archive.
-	if err := tarAddDirectories(tw, path.Dir(name), creationTime); err != nil {
+	// For Windows, the layer must contain a Hives/ directory, and the root
-		return nil, err
+	// of the actual filesystem goes in a Files/ directory.
+	// For Linux, the binary goes into /ko-app/
+	dirs := []string{"ko-app"}
+	if platform.OS == "windows" {
+		dirs = []string{
+			"Hives",
+			"Files",
+			"Files/ko-app",
+		}
+		name = "Files" + name + ".exe"
+	}
+	for _, dir := range dirs {
+		if err := tw.WriteHeader(&tar.Header{
+			Name:     dir,
+			Typeflag: tar.TypeDir,
+			// Use a fixed Mode, so that this isn't sensitive to the directory and umask
+			// under which it was created. Additionally, windows can only set 0222,
+			// 0444, or 0666, none of which are executable.
+			Mode:    0555,
+			ModTime: creationTime.Time,
+		}); err != nil {
+			return nil, fmt.Errorf("writing dir %q: %v", dir, err)
+		}
 	}
 
 	file, err := os.Open(binary)
@@ -483,6 +483,13 @@ func tarBinary(name, binary string, creationTime v1.Time) (*bytes.Buffer, error)
 		Mode:    0555,
 		ModTime: creationTime.Time,
 	}
+	if platform.OS == "windows" {
+		// This magic value is for some reason needed for Windows to be
+		// able to execute the binary.
+		header.PAXRecords = map[string]string{
+			"MSWINDOWS.rawsd": userOwnerAndGroupSID,
+		}
+	}
 	// write the header to the tarball archive
 	if err := tw.WriteHeader(header); err != nil {
 		return nil, err
@@ -509,19 +516,10 @@ const kodataRoot = "/var/run/ko"
 // walkRecursive performs a filepath.Walk of the given root directory adding it
 // to the provided tar.Writer with root -> chroot.  All symlinks are dereferenced,
 // which is what leads to recursion when we encounter a directory symlink.
-func walkRecursive(tw *tar.Writer, root, chroot string, creationTime v1.Time) error {
+func walkRecursive(tw *tar.Writer, root, chroot string, creationTime v1.Time, platform *v1.Platform) error {
 	return filepath.Walk(root, func(hostPath string, info os.FileInfo, err error) error {
 		if hostPath == root {
-			// Add an entry for the root directory of our walk.
+			return nil
-			return tw.WriteHeader(&tar.Header{
-				Name:     chroot,
-				Typeflag: tar.TypeDir,
-				// Use a fixed Mode, so that this isn't sensitive to the directory and umask
-				// under which it was created. Additionally, windows can only set 0222,
-				// 0444, or 0666, none of which are executable.
-				Mode:    0555,
-				ModTime: creationTime.Time,
-			})
 		}
 		if err != nil {
 			return fmt.Errorf("filepath.Walk(%q): %w", root, err)
@@ -532,6 +530,14 @@ func walkRecursive(tw *tar.Writer, root, chroot string, creationTime v1.Time) er
 		}
 		newPath := path.Join(chroot, filepath.ToSlash(hostPath[len(root):]))
 
+		// Don't chase symlinks on Windows, where cross-compiled symlink support is not possible.
+		if platform.OS == "windows" {
+			if info.Mode()&os.ModeSymlink != 0 {
+				log.Println("skipping symlink in kodata for windows:", info.Name())
+				return nil
+			}
+		}
+
 		evalPath, err := filepath.EvalSymlinks(hostPath)
 		if err != nil {
 			return fmt.Errorf("filepath.EvalSymlinks(%q): %w", hostPath, err)
@@ -544,7 +550,7 @@ func walkRecursive(tw *tar.Writer, root, chroot string, creationTime v1.Time) er
 		}
 		// Skip other directories.
 		if info.Mode().IsDir() {
-			return walkRecursive(tw, evalPath, newPath, creationTime)
+			return walkRecursive(tw, evalPath, newPath, creationTime, platform)
 		}
 
 		// Open the file to copy it into the tarball.
@@ -555,7 +561,7 @@ func walkRecursive(tw *tar.Writer, root, chroot string, creationTime v1.Time) er
 		defer file.Close()
 
 		// Copy the file into the image tarball.
-		if err := tw.WriteHeader(&tar.Header{
+		header := &tar.Header{
 			Name:     newPath,
 			Size:     info.Size(),
 			Typeflag: tar.TypeReg,
@@ -564,7 +570,15 @@ func walkRecursive(tw *tar.Writer, root, chroot string, creationTime v1.Time) er
 			// 0444, or 0666, none of which are executable.
 			Mode:    0555,
 			ModTime: creationTime.Time,
-		}); err != nil {
+		}
+		if platform.OS == "windows" {
+			// This magic value is for some reason needed for Windows to be
+			// able to execute the binary.
+			header.PAXRecords = map[string]string{
+				"MSWINDOWS.rawsd": userOwnerAndGroupSID,
+			}
+		}
+		if err := tw.WriteHeader(header); err != nil {
 			return fmt.Errorf("tar.Writer.WriteHeader(%q): %w", newPath, err)
 		}
 		if _, err := io.Copy(tw, file); err != nil {
@@ -574,7 +588,7 @@ func walkRecursive(tw *tar.Writer, root, chroot string, creationTime v1.Time) er
 	})
 }
 
-func (g *gobuild) tarKoData(ref reference) (*bytes.Buffer, error) {
+func (g *gobuild) tarKoData(ref reference, platform *v1.Platform) (*bytes.Buffer, error) {
 	buf := bytes.NewBuffer(nil)
 	tw := tar.NewWriter(buf)
 	defer tw.Close()
@@ -586,7 +600,41 @@ func (g *gobuild) tarKoData(ref reference) (*bytes.Buffer, error) {
 
 	creationTime := g.kodataCreationTime
 
-	return buf, walkRecursive(tw, root, kodataRoot, creationTime)
+	// Write the parent directories to the tarball archive.
+	// For Windows, the layer must contain a Hives/ directory, and the root
+	// of the actual filesystem goes in a Files/ directory.
+	// For Linux, kodata starts at /var/run/ko.
+	chroot := kodataRoot
+	dirs := []string{
+		"/var",
+		"/var/run",
+		"/var/run/ko",
+	}
+	if platform.OS == "windows" {
+		chroot = "Files" + kodataRoot
+		dirs = []string{
+			"Hives",
+			"Files",
+			"Files/var",
+			"Files/var/run",
+			"Files/var/run/ko",
+		}
+	}
+	for _, dir := range dirs {
+		if err := tw.WriteHeader(&tar.Header{
+			Name:     dir,
+			Typeflag: tar.TypeDir,
+			// Use a fixed Mode, so that this isn't sensitive to the directory and umask
+			// under which it was created. Additionally, windows can only set 0222,
+			// 0444, or 0666, none of which are executable.
+			Mode:    0555,
+			ModTime: creationTime.Time,
+		}); err != nil {
+			return nil, fmt.Errorf("writing dir %q: %v", dir, err)
+		}
+	}
+
+	return buf, walkRecursive(tw, root, chroot, creationTime, platform)
 }
 
 func createTemplateData() map[string]interface{} {
@@ -681,8 +729,9 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, baseRef name.Refe
 	defer os.RemoveAll(filepath.Dir(file))
 
 	var layers []mutate.Addendum
+
 	// Create a layer from the kodata directory under this import path.
-	dataLayerBuf, err := g.tarKoData(ref)
+	dataLayerBuf, err := g.tarKoData(ref, platform)
 	if err != nil {
 		return nil, err
 	}
@@ -702,10 +751,10 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, baseRef name.Refe
 		},
 	})
 
-	appPath := path.Join(appDir, appFilename(ref.Path()))
+	appPath := path.Join("/ko-app", appFilename(ref.Path()))
 
 	// Construct a tarball with the binary and produce a layer.
-	binaryLayerBuf, err := tarBinary(appPath, file, v1.Time{})
+	binaryLayerBuf, err := tarBinary(appPath, file, v1.Time{}, platform)
 	if err != nil {
 		return nil, err
 	}
@@ -748,8 +797,14 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, baseRef name.Refe
 
 	cfg = cfg.DeepCopy()
 	cfg.Config.Entrypoint = []string{appPath}
-	updatePath(cfg)
+	if platform.OS == "windows" {
-	cfg.Config.Env = append(cfg.Config.Env, "KO_DATA_PATH="+kodataRoot)
+		cfg.Config.Entrypoint = []string{`C:\ko-app\` + appFilename(ref.Path()) + ".exe"}
+		updatePath(cfg, `C:\ko-app`)
+		cfg.Config.Env = append(cfg.Config.Env, `KO_DATA_PATH=C:\var\run\ko`)
+	} else {
+		updatePath(cfg, appPath)
+		cfg.Config.Env = append(cfg.Config.Env, "KO_DATA_PATH="+kodataRoot)
+	}
 	cfg.Author = "github.com/google/ko"
 
 	if cfg.Config.Labels == nil {
@@ -771,9 +826,9 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, baseRef name.Refe
 	return image, nil
 }
 
-// Append appDir to the PATH environment variable, if it exists. Otherwise,
+// Append appPath to the PATH environment variable, if it exists. Otherwise,
-// set the PATH environment variable to appDir.
+// set the PATH environment variable to appPath.
-func updatePath(cf *v1.ConfigFile) {
+func updatePath(cf *v1.ConfigFile, appPath string) {
 	for i, env := range cf.Config.Env {
 		parts := strings.SplitN(env, "=", 2)
 		if len(parts) != 2 {
@@ -782,14 +837,14 @@ func updatePath(cf *v1.ConfigFile) {
 		}
 		key, value := parts[0], parts[1]
 		if key == "PATH" {
-			value = fmt.Sprintf("%s:%s", value, appDir)
+			value = fmt.Sprintf("%s:%s", value, appPath)
 			cf.Config.Env[i] = "PATH=" + value
 			return
 		}
 	}
 
 	// If we get here, we never saw PATH.
-	cf.Config.Env = append(cf.Config.Env, "PATH="+appDir)
+	cf.Config.Env = append(cf.Config.Env, "PATH="+appPath)
 }
 
 // Build implements build.Interface

pkg/build/gobuild_test.go

@@ -458,7 +458,7 @@ func validateImage(t *testing.T, img v1.Image, baseLayers int64, creationTime v1
 				pathValue := strings.TrimPrefix(envVar, "PATH=")
 				pathEntries := strings.Split(pathValue, ":")
 				for _, pathEntry := range pathEntries {
-					if pathEntry == appDir {
+					if pathEntry == "/ko-app/test" {
 						found = true
 					}
 				}

pkg/publish/daemon.go

@@ -145,7 +145,8 @@ func (d *demon) Publish(ctx context.Context, br build.Result, s string) (name.Re
 	}
 
 	log.Printf("Loading %v", digestTag)
-	if _, err := daemon.Write(digestTag, img, d.getOpts(ctx)...); err != nil {
+	if resp, err := daemon.Write(digestTag, img, d.getOpts(ctx)...); err != nil {
+		log.Println("daemon.Write response: ", resp)
 		return nil, err
 	}
 	log.Printf("Loaded %v", digestTag)

test/kenobi

@@ -1 +0,0 @@
-Hello there

test/kodata/kenobi

@@ -1 +0,0 @@
-../kenobi

test/kodata/kenobi

@@ -0,0 +1 @@
+Hello there

test/main.go

@@ -25,6 +25,7 @@ import (
 )
 
 var (
+	f    = flag.String("f", "kenobi", "File in kodata to print")
 	wait = flag.Bool("wait", true, "Whether to wait for SIGTERM")
 )
 
@@ -32,20 +33,13 @@ func main() {
 	flag.Parse()
 
 	dp := os.Getenv("KO_DATA_PATH")
-	file := filepath.Join(dp, "kenobi")
+	file := filepath.Join(dp, *f)
 	bytes, err := ioutil.ReadFile(file)
 	if err != nil {
 		log.Fatalf("Error reading %q: %v", file, err)
 	}
 	log.Print(string(bytes))
 
-	file = filepath.Join(dp, "HEAD")
-	bytes, err = ioutil.ReadFile(file)
-	if err != nil {
-		log.Fatalf("Error reading %q: %v", file, err)
-	}
-	log.Print(string(bytes))
-
 	// Cause the pod to "hang" to allow us to check for a readiness state.
 	if *wait {
 		sigs := make(chan os.Signal, 1)