diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go
index 7de7280c..07017ec6 100644
--- a/pkg/build/gobuild.go
+++ b/pkg/build/gobuild.go
@@ -702,7 +702,7 @@ func (g *gobuild) configForImportPath(ip string) Config {
 	return config
 }
 
-func (g *gobuild) buildOne(ctx context.Context, refStr string, baseRef name.Reference, baseDigest v1.Hash, base v1.Image, platform *v1.Platform) (v1.Image, error) {
+func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, platform *v1.Platform) (v1.Image, error) {
 	ref := newRef(refStr)
 
 	cf, err := base.ConfigFile()
@@ -779,11 +779,6 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, baseRef name.Refe
 		return nil, err
 	}
 
-	withApp = mutate.Annotations(withApp, map[string]string{
-		specsv1.AnnotationBaseImageDigest: baseDigest.String(),
-		specsv1.AnnotationBaseImageName:   baseRef.Name(),
-	}).(v1.Image)
-
 	// Start from a copy of the base image's config file, and set
 	// the entrypoint to our app.
 	cfg, err := withApp.ConfigFile()
@@ -863,26 +858,44 @@ func (g *gobuild) Build(ctx context.Context, s string) (Result, error) {
 		return nil, err
 	}
 
+	var res Result
 	switch mt {
 	case types.OCIImageIndex, types.DockerManifestList:
 		baseIndex, ok := base.(v1.ImageIndex)
 		if !ok {
 			return nil, fmt.Errorf("failed to interpret base as index: %v", base)
 		}
-		return g.buildAll(ctx, s, baseRef, baseDigest, baseIndex)
+		res, err = g.buildAll(ctx, s, baseIndex)
 	case types.OCIManifestSchema1, types.DockerManifestSchema2:
 		baseImage, ok := base.(v1.Image)
 		if !ok {
 			return nil, fmt.Errorf("failed to interpret base as image: %v", base)
 		}
-		return g.buildOne(ctx, s, baseRef, baseDigest, baseImage, nil)
+		res, err = g.buildOne(ctx, s, baseImage, nil)
 	default:
 		return nil, fmt.Errorf("base image media type: %s", mt)
 	}
+	if err != nil {
+		return nil, err
+	}
+
+	// Annotate the image or index with base image information.
+	// (Docker manifest lists don't support annotations)
+	if mt != types.DockerManifestList {
+		anns := map[string]string{
+			specsv1.AnnotationBaseImageDigest: baseDigest.String(),
+		}
+		if _, ok := baseRef.(name.Tag); ok {
+			anns[specsv1.AnnotationBaseImageName] = baseRef.Name()
+		}
+		res = mutate.Annotations(res, anns).(Result)
+	}
+
+	return res, nil
 }
 
 // TODO(#192): Do these in parallel?
-func (g *gobuild) buildAll(ctx context.Context, ref string, baseRef name.Reference, baseDigest v1.Hash, baseIndex v1.ImageIndex) (v1.ImageIndex, error) {
+func (g *gobuild) buildAll(ctx context.Context, ref string, baseIndex v1.ImageIndex) (v1.ImageIndex, error) {
 	im, err := baseIndex.IndexManifest()
 	if err != nil {
 		return nil, err
@@ -904,7 +917,7 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseRef name.Referen
 		if err != nil {
 			return nil, err
 		}
-		img, err := g.buildOne(ctx, ref, baseRef, baseDigest, baseImage, desc.Platform)
+		img, err := g.buildOne(ctx, ref, baseImage, desc.Platform)
 		if err != nil {
 			return nil, err
 		}
@@ -925,15 +938,6 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseRef name.Referen
 	}
 	idx := mutate.IndexMediaType(mutate.AppendManifests(empty.Index, adds...), baseType)
 
-	// Annotate the index with base image information, if the index is an OCI image index.
-	// (Docker manifest lists don't support annotations)
-	if baseType == types.OCIImageIndex {
-		idx = mutate.Annotations(idx, map[string]string{
-			specsv1.AnnotationBaseImageName:   baseRef.Name(),
-			specsv1.AnnotationBaseImageDigest: baseDigest.String(),
-		}).(v1.ImageIndex)
-	}
-
 	return idx, nil
 }
 
diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go
index a8b7eef1..8f8211f5 100644
--- a/pkg/build/gobuild_test.go
+++ b/pkg/build/gobuild_test.go
@@ -343,7 +343,7 @@ func TestGoBuildNoKoData(t *testing.T) {
 	})
 }
 
-func validateImage(t *testing.T, img v1.Image, baseLayers int64, creationTime v1.Time) {
+func validateImage(t *testing.T, img v1.Image, baseLayers int64, creationTime v1.Time, checkAnnotations bool) {
 	t.Helper()
 
 	ls, err := img.Layers()
@@ -483,6 +483,9 @@ func validateImage(t *testing.T, img v1.Image, baseLayers int64, creationTime v1
 	})
 
 	t.Run("check annotations", func(t *testing.T) {
+		if !checkAnnotations {
+			t.Skip("skipping annotations check")
+		}
 		mf, err := img.Manifest()
 		if err != nil {
 			t.Fatalf("Manifest() = %v", err)
@@ -540,7 +543,7 @@ func TestGoBuild(t *testing.T) {
 		t.Fatalf("Build() not an image: %v", result)
 	}
 
-	validateImage(t, img, baseLayers, creationTime)
+	validateImage(t, img, baseLayers, creationTime, true)
 
 	// Check that rebuilding the image again results in the same image digest.
 	t.Run("check determinism", func(t *testing.T) {
@@ -622,7 +625,7 @@ func TestGoBuildIndex(t *testing.T) {
 		if err != nil {
 			t.Fatalf("idx.Image(%s) = %v", desc.Digest, err)
 		}
-		validateImage(t, img, baseLayers, creationTime)
+		validateImage(t, img, baseLayers, creationTime, false)
 	}
 
 	if want, got := images, int64(len(im.Manifests)); want != got {