Generate semantic conventions according to specification latest tagged version (#1933)

This modifies the generation tool to generate the content from the latest tagged
version of the specification (previously it was just using whatever the spec repo's
working directory contained).

The generated files will be now placed in a sub-directory with a name matching the
tagged version number. The generated package will be placed under the "semconv" directory.

Note: we will use the new logic generate the semconv/v1.3.0 package in a separate future PR.

RELEASING.md

@@ -4,9 +4,10 @@
 
 If a new version of the OpenTelemetry Specification has been released it will be necessary to generate a new
 semantic convention package from the YAML definitions in the specification repository. There is a utility in
-`internal/tools/semconv-gen` that can be used to generate the `semconv` package. This will ideally be done
-shortly after the specification release is tagged, but it is also good practice to ensure that current conventions
-are current before creating a release tag.
+`internal/tools/semconv-gen` that can be used to generate the a package with the name matching the specification
+version number under the `semconv` package. This will ideally be done soon after the specification release is
+tagged. Make sure that the specification repo contains a checkout of the the latest tagged release so that the
+generated files match the released semantic conventions.
 
 There are currently two categories of semantic conventions that must be generated, `resource` and `trace`.
 
@@ -17,7 +18,7 @@ go run generate.go -i /path/to/specification/repo/semantic_conventions/trace
 ```
 
 Using default values for all options other than `input` will result in using the `template.j2` template to
-generate `resource.go` and `trace.go` in `/path/to/otelgo/repo/semconv`.
+generate `resource.go` and `trace.go` in `/path/to/otelgo/repo/semconv/<version>`.
 
 ## Pre-Release
 

internal/tools/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/golangci/golangci-lint v1.40.1
 	github.com/itchyny/gojq v0.12.3
 	github.com/spf13/pflag v1.0.5
+	golang.org/x/mod v0.4.2
 	golang.org/x/tools v0.1.2-0.20210512205948-8287d5da45e4
 )
 

internal/tools/semconv-gen/generator.go

@@ -19,20 +19,27 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 
 	flag "github.com/spf13/pflag"
+	"golang.org/x/mod/semver"
 )
 
 func main() {
+	// Plain log output, no timestamps.
+	log.SetFlags(0)
+
 	cfg := config{}
-	flag.StringVarP(&cfg.inputPath, "input", "i", "", "Path to semantic convention definition YAML")
-	flag.StringVarP(&cfg.outputPath, "output", "o", "semconv", "Path to output target. Must be either an absolute path or relative to the repository root.")
+	flag.StringVarP(&cfg.inputPath, "input", "i", "", "Path to semantic convention definition YAML. Should be a directory in the specification git repository.")
+	flag.StringVarP(&cfg.specVersion, "specver", "s", "", "Version of semantic convention to generate. Must be an existing version tag in the specification git repository.")
+	flag.StringVarP(&cfg.outputPath, "output", "o", "", "Path to output target. Must be either an absolute path or relative to the repository root. If unspecified will output to a sub-directory with the name matching the version number specified via --specver flag.")
 	flag.StringVarP(&cfg.containerImage, "container", "c", "otel/semconvgen", "Container image ID")
 	flag.StringVarP(&cfg.outputFilename, "filename", "f", "", "Filename for templated output. If not specified 'basename(inputPath).go' will be used.")
 	flag.StringVarP(&cfg.templateFilename, "template", "t", "template.j2", "Template filename")
@@ -50,7 +57,7 @@ func main() {
 		panic(err)
 	}
 
-	err = fixIdentifiers(cfg.outputFilename)
+	err = fixIdentifiers(cfg)
 	if err != nil {
 		panic(err)
 	}
@@ -67,17 +74,29 @@ type config struct {
 	outputFilename   string
 	templateFilename string
 	containerImage   string
+	specVersion      string
 }
 
 func validateConfig(cfg config) (config, error) {
-	if cfg.inputPath == "" {
-		return config{}, errors.New("input path must be provided")
-	}
-
 	if cfg.outputFilename == "" {
 		cfg.outputFilename = fmt.Sprintf("%s.go", path.Base(cfg.inputPath))
 	}
 
+	if cfg.specVersion == "" {
+		// Find the latest version of the specification and use it for generation.
+		var err error
+		cfg.specVersion, err = findLatestSpecVersion(cfg)
+		if err != nil {
+			return config{}, err
+		}
+	}
+
+	if cfg.outputPath == "" {
+		// If output path is unspecified put it under a sub-directory with a name matching
+		// the version of semantic convention under the semconv directory.
+		cfg.outputPath = path.Join("semconv", cfg.specVersion)
+	}
+
 	if !path.IsAbs(cfg.outputPath) {
 		root, err := findRepoRoot()
 		if err != nil {
@@ -106,8 +125,8 @@ func render(cfg config) error {
 	}
 	defer os.RemoveAll(tmpDir)
 
-	inputPath := path.Join(tmpDir, "input")
-	err = os.Mkdir(inputPath, 0700)
+	specCheckoutPath := path.Join(tmpDir, "input")
+	err = os.Mkdir(specCheckoutPath, 0700)
 	if err != nil {
 		return fmt.Errorf("unable to create input directory: %w", err)
 	}
@@ -118,10 +137,13 @@ func render(cfg config) error {
 		return fmt.Errorf("unable to create output directory: %w", err)
 	}
 
-	err = exec.Command("cp", "-a", cfg.inputPath, inputPath).Run()
+	// Checkout the specification repo to a temp dir. This will be the input
+	// for the generator.
+	doneFunc, err := checkoutSpecToDir(cfg, specCheckoutPath)
 	if err != nil {
-		return fmt.Errorf("unable to copy input to temp directory: %w", err)
+		return err
 	}
+	defer doneFunc()
 
 	err = exec.Command("cp", cfg.templateFilename, tmpDir).Run()
 	if err != nil {
@@ -131,7 +153,7 @@ func render(cfg config) error {
 	cmd := exec.Command("docker", "run", "--rm",
 		"-v", fmt.Sprintf("%s:/data", tmpDir),
 		cfg.containerImage,
-		"--yaml-root", path.Join("/data/input", path.Base(cfg.inputPath)),
+		"--yaml-root", path.Join("/data/input/semantic_conventions/", path.Base(cfg.inputPath)),
 		"code",
 		"--template", path.Join("/data", path.Base(cfg.templateFilename)),
 		"--output", path.Join("/data/output", path.Base(cfg.outputFilename)),
@@ -141,6 +163,10 @@ func render(cfg config) error {
 		return fmt.Errorf("unable to render template: %w", err)
 	}
 
+	err = os.MkdirAll(cfg.outputPath, 0700)
+	if err != nil {
+		return fmt.Errorf("unable to create output directory %s: %w", cfg.outputPath, err)
+	}
 	err = exec.Command("cp", path.Join(tmpDir, "output", path.Base(cfg.outputFilename)), cfg.outputPath).Run()
 	if err != nil {
 		return fmt.Errorf("unable to copy result to target: %w", err)
@@ -149,6 +175,84 @@ func render(cfg config) error {
 	return nil
 }
 
+type semVerSlice []string
+
+func (s semVerSlice) Len() int {
+	return len(s)
+}
+
+func (s semVerSlice) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s semVerSlice) Less(i, j int) bool {
+	return semver.Compare(s[i], s[j]) < 0
+}
+
+// findLatestSpecVersion finds the latest specification version number and checkouts
+// that version in the repo's working directory.
+func findLatestSpecVersion(cfg config) (string, error) {
+	// List all tags in the specification repo. All released version numbers are tags
+	// in the repo.
+	cmd := exec.Command("git", "tag")
+	// The specification repo is in cfg.inputPath.
+	cmd.Dir = cfg.inputPath
+	output, err := cmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("unable to exec %s: %w", cmd.String(), err)
+	}
+
+	// Split the output: each line is a tag.
+	lines := strings.Split(string(output), "\n")
+
+	// Copy valid semver version numbers to a slice.
+	var versions semVerSlice
+	for _, line := range lines {
+		ver := line
+		if semver.IsValid(ver) {
+			versions = append(versions, ver)
+		}
+	}
+
+	// Sort it according to semver rules.
+	sort.Sort(versions)
+
+	if len(versions) == 0 {
+		return "", fmt.Errorf("no version tags found in the specification repo at %s", cfg.inputPath)
+	}
+
+	// Use the latest version number.
+	lastVer := versions[len(versions)-1]
+	return lastVer, nil
+}
+
+// checkoutSpecToDir checks out the specification repository to the toDir.
+// Returned doneFunc should be called when the directory is no longer needed and can be
+// cleaned up.
+func checkoutSpecToDir(cfg config, toDir string) (doneFunc func(), err error) {
+	// Checkout the selected tag to make sure we use the correct version of semantic
+	// convention yaml files as the input. We will checkout the worktree to a temporary toDir.
+	cmd := exec.Command("git", "worktree", "add", toDir, cfg.specVersion)
+	// The specification repo is in cfg.inputPath.
+	cmd.Dir = cfg.inputPath
+	err = cmd.Run()
+	if err != nil {
+		return nil, fmt.Errorf("unable to exec %s: %w", cmd.String(), err)
+	}
+
+	doneFunc = func() {
+		// Remove the worktree when it is no longer needed.
+		cmd := exec.Command("git", "worktree", "remove", "-f", toDir)
+		cmd.Dir = cfg.inputPath
+		err := cmd.Run()
+		if err != nil {
+			log.Printf("Could not cleanup spec repo worktree, unable to exec %s: %s\n", cmd.String(), err.Error())
+		}
+	}
+
+	return doneFunc, nil
+}
+
 func findRepoRoot() (string, error) {
 	start, err := os.Getwd()
 	if err != nil {
@@ -277,8 +381,8 @@ var replacements = map[string]string{
 	"Lineno":        "LineNumber",
 }
 
-func fixIdentifiers(fn string) error {
-	data, err := ioutil.ReadFile(fn)
+func fixIdentifiers(cfg config) error {
+	data, err := ioutil.ReadFile(cfg.outputFilename)
 	if err != nil {
 		return fmt.Errorf("unable to read file: %w", err)
 	}
@@ -298,7 +402,12 @@ func fixIdentifiers(fn string) error {
 		data = bytes.ReplaceAll(data, []byte(cur), []byte(repl))
 	}
 
-	err = ioutil.WriteFile(fn, data, 0644)
+	// Inject the correct import path.
+	packageDir := path.Base(path.Dir(cfg.outputFilename))
+	importPath := fmt.Sprintf(`"go.opentelemetry.io/otel/semconv/%s"`, packageDir)
+	data = bytes.ReplaceAll(data, []byte(`[[IMPORTPATH]]`), []byte(importPath))
+
+	err = ioutil.WriteFile(cfg.outputFilename, data, 0644)
 	if err != nil {
 		return fmt.Errorf("unable to write updated file: %w", err)
 	}

internal/tools/semconv-gen/template.j2

@@ -46,7 +46,7 @@ Note: {{ attr.note }}
 
 // Code generated from semantic convention specification. DO NOT EDIT.
 
-package semconv  // import "go.opentelemetry.io/otel/semconv"
+package semconv  // import [[IMPORTPATH]]
 
 import "go.opentelemetry.io/otel/attribute"