Add migration doc generation to semconvgen (#6819)

Fix #6248

Makefile

@@ -293,7 +293,7 @@ semconv-generate: $(SEMCONVKIT)
 		--param tag=$(TAG) \
 		go \
 		/home/weaver/target
-	$(SEMCONVKIT) -output "$(SEMCONVPKG)/$(TAG)" -tag "$(TAG)"
+	$(SEMCONVKIT) -semconv "$(SEMCONVPKG)" -tag "$(TAG)"
 
 .PHONY: gorelease
 gorelease: $(OTEL_GO_MOD_DIRS:%=gorelease/%)

internal/tools/go.mod

@@ -3,6 +3,7 @@ module go.opentelemetry.io/otel/internal/tools
 go 1.23.0
 
 require (
+	github.com/Masterminds/semver v1.5.0
 	github.com/client9/misspell v0.3.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/golangci/golangci-lint/v2 v2.1.6

internal/tools/go.sum

@@ -20,6 +20,8 @@ github.com/Djarvur/go-err113 v0.1.0 h1:uCRZZOdMQ0TZPHYTdYpoC0bLYJKPEHPUJ8MeAa51l
 github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
 github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1 h1:Sz1JIXEcSfhz7fUi7xHnhpIE0thVASYjvosApmHuD2k=
 github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1/go.mod h1:n/LSCXNuIYqVfBlVXyHfMQkZDdp1/mmxfSjADd3z1Zg=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
 github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=

internal/tools/semconvkit/decls/decls.go

@@ -0,0 +1,58 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Package decls provides a set of functions to parse and analyze Go source
+// code and get the declarations within it.
+package decls // import "go.opentelemetry.io/otel/internal/tools/semconvkit/decls"
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"strings"
+)
+
+// GetNames parses the Go source code in the specified package path and returns
+// the names extracted from the declarations using the provided parser
+// function.
+//
+// The names are returned as a map where the keys are the names fully
+// lowercased form of the name and the values are the original format of the
+// name.
+func GetNames(pkgPath string, f Parser) (Names, error) {
+	fset := token.NewFileSet()
+	pkgs, err := parser.ParseDir(fset, pkgPath, nil, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(Names)
+	for _, pkg := range pkgs {
+		for _, file := range pkg.Files {
+			for _, decl := range file.Decls {
+				for _, name := range f(decl) {
+					out[NewCanonicalName(name)] = Name(name)
+				}
+			}
+		}
+	}
+	return out, nil
+}
+
+// Parser is a function type that takes an [ast.Decl] and returns a slice of
+// parsed string identifiers.
+type Parser func(ast.Decl) []string
+
+// CanonicalName is the canonical form of a name (lowercase).
+type CanonicalName string
+
+// NewCanonicalName returns name as a [CanonicalName].
+func NewCanonicalName(name string) CanonicalName {
+	return CanonicalName(strings.ToLower(name))
+}
+
+// Name is the original form of a name (case-sensitive).
+type Name string
+
+// Names is a map of canonical names to their original names.
+type Names map[CanonicalName]Name

internal/tools/semconvkit/main.go

@@ -2,47 +2,119 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // Package semconvkit is used to generate opentelemetry-go specific semantic
-// convention code. It is expected to be used in with the semconvgen utility
-// (go.opentelemetry.io/build-tools/semconvgen) to completely generate
-// versioned sub-packages of go.opentelemetry.io/otel/semconv.
+// convention code.
 package main
 
 import (
 	"embed"
+	"errors"
 	"flag"
-	"log"
+	"fmt"
+	"go/ast"
+	"go/token"
+	"log/slog"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"text/template"
+
+	"github.com/Masterminds/semver"
+	"go.opentelemetry.io/otel/internal/tools/semconvkit/decls"
 )
 
 var (
-	out = flag.String("output", "./", "output directory")
-	tag = flag.String("tag", "", "OpenTelemetry tagged version")
+	logLevel   = flag.String("log-level", "", `Logging level ("debug", "info", "warn", "error")`)
+	semconvPkg = flag.String("semconv", "./", "semconv package directory")
+	tag        = flag.String("tag", "", "OpenTelemetry tagged version")
+	prev       = flag.String("prev", "", "previous semconv version")
 
 	//go:embed templates/*.tmpl
 	rootFS embed.FS
 )
 
-// SemanticConventions are information about the semantic conventions being
-// generated.
-type SemanticConventions struct {
-	// TagVer is the tagged version (i.e. v1.7.0 and not 1.7.0).
-	TagVer string
+func main() {
+	flag.Parse()
+
+	slog.SetDefault(newLogger(*logLevel))
+
+	if *tag == "" {
+		slog.Error("invalid tag", "tag", *tag)
+		os.Exit(1)
+	}
+
+	sc := &SemanticConventions{TagVer: *tag}
+
+	out := filepath.Join(*semconvPkg, *tag)
+
+	// Render all other files before the MIGRATION file. That file needs the
+	// full package declaration so it can determine compatibility accurately.
+	entries, err := rootFS.ReadDir("templates")
+	if err != nil {
+		slog.Error("error reading templates", "err", err)
+		os.Exit(1)
+	}
+
+	for _, entry := range entries {
+		if entry.Name() == "MIGRATION.md.tmpl" {
+			continue
+		}
+
+		src := filepath.Join("templates", entry.Name())
+		err := render(src, out, sc)
+		if err != nil {
+			slog.Error("error rendering template", "err", err, "template", entry.Name())
+			os.Exit(1)
+		}
+	}
+
+	prevPkg, err := prevVer(*semconvPkg, *tag, *prev)
+	if err != nil {
+		slog.Error("previous version not found, skipping migration", "err", err)
+		os.Exit(1)
+	}
+
+	slog.Debug("previous version found", "prev", prevPkg)
+	m, err := newMigration(out, filepath.Join(*semconvPkg, prevPkg))
+	if err != nil {
+		slog.Error("error getting migration, skipping", "err", err)
+		os.Exit(1)
+	}
+
+	if err := render("templates/MIGRATION.md.tmpl", out, m); err != nil {
+		slog.Error("error rendering migration template", "err", err)
+		os.Exit(1)
+	}
 }
 
-func (sc SemanticConventions) SemVer() string {
-	return strings.TrimPrefix(*tag, "v")
+func newLogger(lvlStr string) *slog.Logger {
+	levelVar := new(slog.LevelVar) // Default value of info.
+	opts := &slog.HandlerOptions{AddSource: true, Level: levelVar}
+	h := slog.NewTextHandler(os.Stderr, opts)
+	logger := slog.New(h)
+
+	if lvlStr == "" {
+		return logger
+	}
+
+	var level slog.Level
+	if err := level.UnmarshalText([]byte(lvlStr)); err != nil {
+		logger.Error("failed to parse log level", "error", err, "log-level", lvlStr)
+	} else {
+		levelVar.Set(level)
+	}
+
+	return logger
 }
 
 // render renders all templates to the dest directory using the data.
-func render(src, dest string, data *SemanticConventions) error {
+func render(src, dest string, data any) error {
 	tmpls, err := template.ParseFS(rootFS, src)
 	if err != nil {
 		return err
 	}
 	for _, tmpl := range tmpls.Templates() {
+		slog.Debug("rendering template", "name", tmpl.Name())
 		target := filepath.Join(dest, strings.TrimSuffix(tmpl.Name(), ".tmpl"))
 		wr, err := os.Create(target)
 		if err != nil {
@@ -58,16 +130,156 @@ func render(src, dest string, data *SemanticConventions) error {
 	return nil
 }
 
-func main() {
-	flag.Parse()
-
-	if *tag == "" {
-		log.Fatalf("invalid tag: %q", *tag)
+// prevVer returns the previous version of the semantic conventions package.
+// It will first check for hint within root and return that value if found. If
+// not found, it will find all directories in root with a version name and
+// return the version that is less than and closest to the curr version.
+func prevVer(root, cur, hint string) (string, error) {
+	slog.Debug("prevVer", "root", root, "current", cur, "hint", hint)
+	info, err := os.Stat(root)
+	if err != nil {
+		return "", fmt.Errorf("root directory %q not found: %w", root, err)
+	}
+	if !info.IsDir() {
+		return "", fmt.Errorf("root %q is not a directory", root)
 	}
 
-	sc := &SemanticConventions{TagVer: *tag}
-
-	if err := render("templates/*.tmpl", *out, sc); err != nil {
-		log.Fatal(err)
+	if hint != "" {
+		sub := filepath.Join(root, hint)
+		slog.Debug("looking for hint", "path", sub)
+		info, err = os.Stat(sub)
+		if err == nil && info.IsDir() {
+			return hint, nil
+		}
 	}
+
+	v, err := semver.NewVersion(cur)
+	if err != nil {
+		return "", fmt.Errorf("invalid current version %q: %w", cur, err)
+	}
+
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		return "", fmt.Errorf("error reading root %q: %w", root, err)
+	}
+
+	var prev *semver.Version
+	for _, entry := range entries {
+		slog.Debug("root entry", "name", entry.Name())
+		if !entry.IsDir() {
+			continue
+		}
+
+		ver, err := semver.NewVersion(entry.Name())
+		if err != nil {
+			slog.Debug("not a version dir", "name", entry.Name())
+			// Ignore errors for non-semver directories.
+			continue
+		}
+		slog.Debug("found version dir", "prev", ver)
+		if ver.LessThan(v) && (prev == nil || ver.GreaterThan(prev)) {
+			slog.Debug("new previous version", "version", ver)
+			prev = ver
+		}
+	}
+
+	if prev == nil {
+		return "", errors.New("no previous version found")
+	}
+	return prev.Original(), nil
+}
+
+// SemanticConventions are information about the semantic conventions being
+// generated.
+type SemanticConventions struct {
+	// TagVer is the tagged version (i.e. v.7.0 and not 1.7.0).
+	TagVer string
+}
+
+func (sc SemanticConventions) SemVer() string {
+	return strings.TrimPrefix(*tag, "v")
+}
+
+// Migration contains the details about the migration from the previous
+// semantic conventions to the current one.
+type Migration struct {
+	CurVer   string
+	PrevVer  string
+	Removals []string
+	Renames  []Rename
+}
+
+// Remove is a semantic convention declaration that has been renamed.
+type Rename struct {
+	Old, New string
+}
+
+func newMigration(cur, prev string) (*Migration, error) {
+	cDecl, err := decls.GetNames(cur, parse)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing current version %q: %w", cur, err)
+	}
+
+	pDecl, err := decls.GetNames(prev, parse)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing previous version %q: %w", prev, err)
+	}
+
+	m := Migration{
+		CurVer:   filepath.Base(cur),
+		PrevVer:  filepath.Base(prev),
+		Removals: inAnotB(pDecl, cDecl),
+		Renames:  renames(pDecl, cDecl),
+	}
+
+	sort.Strings(m.Removals)
+	sort.Slice(m.Renames, func(i, j int) bool {
+		return m.Renames[i].Old < m.Renames[j].Old
+	})
+
+	return &m, nil
+}
+
+func parse(d ast.Decl) []string {
+	var out []string
+	switch decl := d.(type) {
+	case *ast.FuncDecl:
+		out = []string{decl.Name.Name}
+	case *ast.GenDecl:
+		if decl.Tok == token.CONST || decl.Tok == token.VAR {
+			for _, spec := range decl.Specs {
+				if valueSpec, ok := spec.(*ast.ValueSpec); ok {
+					for _, name := range valueSpec.Names {
+						out = append(out, name.Name)
+					}
+				}
+			}
+		}
+	}
+	return out
+}
+
+// inAnotB returns the canonical names in a that are not in b.
+func inAnotB(a, b decls.Names) []string {
+	var diff []string
+	for key, name := range a {
+		if _, ok := b[key]; !ok {
+			diff = append(diff, string(name))
+		}
+	}
+	return diff
+}
+
+// renames returns the renames between the old and current names.
+func renames(old, current decls.Names) []Rename {
+	var renames []Rename
+	for key, name := range old {
+		if otherName, ok := current[key]; ok && name != otherName {
+			renames = append(renames, Rename{
+				Old: string(name),
+				New: string(otherName),
+			})
+		}
+	}
+	return renames
 }

internal/tools/semconvkit/templates/MIGRATION.md.tmpl

@@ -0,0 +1,40 @@
+<!-- Generated. DO NOT MODIFY. -->
+{{ if .PrevVer -}} 
+# Migration from {{ .PrevVer }} to {{ .CurVer }}
+{{ if or .Removals .Renames }}
+The `go.opentelemetry.io/otel/semconv/{{ .CurVer }}` package should be a drop-in replacement for `go.opentelemetry.io/otel/semconv/{{ .PrevVer }}` with the following exceptions.
+{{ if .Renames }}
+## Renames
+
+The following renames have been introduced to better match Go and industry naming standards.
+Be sure to update any use from `go.opentelemetry.io/otel/semconv/{{ .PrevVer }}` with the equivalent in `go.opentelemetry.io/otel/semconv/{{ .CurVer }}`.
+
+| `{{ .PrevVer }}` | `{{ .CurVer }}` |
+| --- | --- |
+{{ range .Renames -}}
+| `{{ .Old }}` | `{{ .New }}` |
+{{ end -}}
+{{ end -}}
+{{ if .Removals }}
+## Removed
+
+The following declarations have been removed.
+Refer to the [OpenTelemetry Semantic Conventions documentation] for deprecation instructions.
+
+If the type is not listed in the documentation as deprecated, it has been removed in this version due to lack of applicability or use.
+If you use any of these non-deprecated declarations in your Go application, please [open an issue] describing your use-case.
+
+{{ range .Removals -}}
+- `{{ . }}`
+{{ end }}
+[OpenTelemetry Semantic Conventions documentation]: https://github.com/open-telemetry/semantic-conventions
+[open an issue]: https://github.com/open-telemetry/opentelemetry-go/issues/new?template=Blank+issue
+{{ end -}}
+{{ else }}
+The `go.opentelemetry.io/otel/semconv/{{ .CurVer }}` package should be a drop-in replacement for `go.opentelemetry.io/otel/semconv/{{ .PrevVer }}`.
+{{ end -}}
+{{ else }}
+# {{ .CurVer }} Migration
+
+The `go.opentelemetry.io/otel/semconv/{{ .CurVer }}` package should be a drop-in replacement for previous versions of the prior `go.opentelemetry.io/otel/semconv`.
+{{ end -}}

semconv/v1.34.0/MIGRATION.md

@@ -0,0 +1,4 @@
+<!-- Generated. DO NOT MODIFY. -->
+# Migration from v1.33.0 to v1.34.0
+
+The `go.opentelemetry.io/otel/semconv/v1.34.0` package should be a drop-in replacement for `go.opentelemetry.io/otel/semconv/v1.33.0`.