diff --git a/RELEASING.md b/RELEASING.md index 29f8ce7d7..7a094726f 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -4,9 +4,10 @@ If a new version of the OpenTelemetry Specification has been released it will be necessary to generate a new semantic convention package from the YAML definitions in the specification repository. There is a utility in -`internal/tools/semconv-gen` that can be used to generate the `semconv` package. This will ideally be done -shortly after the specification release is tagged, but it is also good practice to ensure that current conventions -are current before creating a release tag. +`internal/tools/semconv-gen` that can be used to generate the a package with the name matching the specification +version number under the `semconv` package. This will ideally be done soon after the specification release is +tagged. Make sure that the specification repo contains a checkout of the the latest tagged release so that the +generated files match the released semantic conventions. There are currently two categories of semantic conventions that must be generated, `resource` and `trace`. @@ -17,7 +18,7 @@ go run generate.go -i /path/to/specification/repo/semantic_conventions/trace ``` Using default values for all options other than `input` will result in using the `template.j2` template to -generate `resource.go` and `trace.go` in `/path/to/otelgo/repo/semconv`. +generate `resource.go` and `trace.go` in `/path/to/otelgo/repo/semconv/`. ## Pre-Release diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 19c78ce31..51fcda082 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -8,6 +8,7 @@ require ( github.com/golangci/golangci-lint v1.40.1 github.com/itchyny/gojq v0.12.3 github.com/spf13/pflag v1.0.5 + golang.org/x/mod v0.4.2 golang.org/x/tools v0.1.2-0.20210512205948-8287d5da45e4 ) diff --git a/internal/tools/semconv-gen/generator.go b/internal/tools/semconv-gen/generator.go index 7c44aaf37..702709271 100644 --- a/internal/tools/semconv-gen/generator.go +++ b/internal/tools/semconv-gen/generator.go @@ -19,20 +19,27 @@ import ( "errors" "fmt" "io/ioutil" + "log" "os" "os/exec" "path" "path/filepath" "regexp" + "sort" "strings" flag "github.com/spf13/pflag" + "golang.org/x/mod/semver" ) func main() { + // Plain log output, no timestamps. + log.SetFlags(0) + cfg := config{} - flag.StringVarP(&cfg.inputPath, "input", "i", "", "Path to semantic convention definition YAML") - flag.StringVarP(&cfg.outputPath, "output", "o", "semconv", "Path to output target. Must be either an absolute path or relative to the repository root.") + flag.StringVarP(&cfg.inputPath, "input", "i", "", "Path to semantic convention definition YAML. Should be a directory in the specification git repository.") + flag.StringVarP(&cfg.specVersion, "specver", "s", "", "Version of semantic convention to generate. Must be an existing version tag in the specification git repository.") + flag.StringVarP(&cfg.outputPath, "output", "o", "", "Path to output target. Must be either an absolute path or relative to the repository root. If unspecified will output to a sub-directory with the name matching the version number specified via --specver flag.") flag.StringVarP(&cfg.containerImage, "container", "c", "otel/semconvgen", "Container image ID") flag.StringVarP(&cfg.outputFilename, "filename", "f", "", "Filename for templated output. If not specified 'basename(inputPath).go' will be used.") flag.StringVarP(&cfg.templateFilename, "template", "t", "template.j2", "Template filename") @@ -50,7 +57,7 @@ func main() { panic(err) } - err = fixIdentifiers(cfg.outputFilename) + err = fixIdentifiers(cfg) if err != nil { panic(err) } @@ -67,17 +74,29 @@ type config struct { outputFilename string templateFilename string containerImage string + specVersion string } func validateConfig(cfg config) (config, error) { - if cfg.inputPath == "" { - return config{}, errors.New("input path must be provided") - } - if cfg.outputFilename == "" { cfg.outputFilename = fmt.Sprintf("%s.go", path.Base(cfg.inputPath)) } + if cfg.specVersion == "" { + // Find the latest version of the specification and use it for generation. + var err error + cfg.specVersion, err = findLatestSpecVersion(cfg) + if err != nil { + return config{}, err + } + } + + if cfg.outputPath == "" { + // If output path is unspecified put it under a sub-directory with a name matching + // the version of semantic convention under the semconv directory. + cfg.outputPath = path.Join("semconv", cfg.specVersion) + } + if !path.IsAbs(cfg.outputPath) { root, err := findRepoRoot() if err != nil { @@ -106,8 +125,8 @@ func render(cfg config) error { } defer os.RemoveAll(tmpDir) - inputPath := path.Join(tmpDir, "input") - err = os.Mkdir(inputPath, 0700) + specCheckoutPath := path.Join(tmpDir, "input") + err = os.Mkdir(specCheckoutPath, 0700) if err != nil { return fmt.Errorf("unable to create input directory: %w", err) } @@ -118,10 +137,13 @@ func render(cfg config) error { return fmt.Errorf("unable to create output directory: %w", err) } - err = exec.Command("cp", "-a", cfg.inputPath, inputPath).Run() + // Checkout the specification repo to a temp dir. This will be the input + // for the generator. + doneFunc, err := checkoutSpecToDir(cfg, specCheckoutPath) if err != nil { - return fmt.Errorf("unable to copy input to temp directory: %w", err) + return err } + defer doneFunc() err = exec.Command("cp", cfg.templateFilename, tmpDir).Run() if err != nil { @@ -131,7 +153,7 @@ func render(cfg config) error { cmd := exec.Command("docker", "run", "--rm", "-v", fmt.Sprintf("%s:/data", tmpDir), cfg.containerImage, - "--yaml-root", path.Join("/data/input", path.Base(cfg.inputPath)), + "--yaml-root", path.Join("/data/input/semantic_conventions/", path.Base(cfg.inputPath)), "code", "--template", path.Join("/data", path.Base(cfg.templateFilename)), "--output", path.Join("/data/output", path.Base(cfg.outputFilename)), @@ -141,6 +163,10 @@ func render(cfg config) error { return fmt.Errorf("unable to render template: %w", err) } + err = os.MkdirAll(cfg.outputPath, 0700) + if err != nil { + return fmt.Errorf("unable to create output directory %s: %w", cfg.outputPath, err) + } err = exec.Command("cp", path.Join(tmpDir, "output", path.Base(cfg.outputFilename)), cfg.outputPath).Run() if err != nil { return fmt.Errorf("unable to copy result to target: %w", err) @@ -149,6 +175,84 @@ func render(cfg config) error { return nil } +type semVerSlice []string + +func (s semVerSlice) Len() int { + return len(s) +} + +func (s semVerSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s semVerSlice) Less(i, j int) bool { + return semver.Compare(s[i], s[j]) < 0 +} + +// findLatestSpecVersion finds the latest specification version number and checkouts +// that version in the repo's working directory. +func findLatestSpecVersion(cfg config) (string, error) { + // List all tags in the specification repo. All released version numbers are tags + // in the repo. + cmd := exec.Command("git", "tag") + // The specification repo is in cfg.inputPath. + cmd.Dir = cfg.inputPath + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("unable to exec %s: %w", cmd.String(), err) + } + + // Split the output: each line is a tag. + lines := strings.Split(string(output), "\n") + + // Copy valid semver version numbers to a slice. + var versions semVerSlice + for _, line := range lines { + ver := line + if semver.IsValid(ver) { + versions = append(versions, ver) + } + } + + // Sort it according to semver rules. + sort.Sort(versions) + + if len(versions) == 0 { + return "", fmt.Errorf("no version tags found in the specification repo at %s", cfg.inputPath) + } + + // Use the latest version number. + lastVer := versions[len(versions)-1] + return lastVer, nil +} + +// checkoutSpecToDir checks out the specification repository to the toDir. +// Returned doneFunc should be called when the directory is no longer needed and can be +// cleaned up. +func checkoutSpecToDir(cfg config, toDir string) (doneFunc func(), err error) { + // Checkout the selected tag to make sure we use the correct version of semantic + // convention yaml files as the input. We will checkout the worktree to a temporary toDir. + cmd := exec.Command("git", "worktree", "add", toDir, cfg.specVersion) + // The specification repo is in cfg.inputPath. + cmd.Dir = cfg.inputPath + err = cmd.Run() + if err != nil { + return nil, fmt.Errorf("unable to exec %s: %w", cmd.String(), err) + } + + doneFunc = func() { + // Remove the worktree when it is no longer needed. + cmd := exec.Command("git", "worktree", "remove", "-f", toDir) + cmd.Dir = cfg.inputPath + err := cmd.Run() + if err != nil { + log.Printf("Could not cleanup spec repo worktree, unable to exec %s: %s\n", cmd.String(), err.Error()) + } + } + + return doneFunc, nil +} + func findRepoRoot() (string, error) { start, err := os.Getwd() if err != nil { @@ -277,8 +381,8 @@ var replacements = map[string]string{ "Lineno": "LineNumber", } -func fixIdentifiers(fn string) error { - data, err := ioutil.ReadFile(fn) +func fixIdentifiers(cfg config) error { + data, err := ioutil.ReadFile(cfg.outputFilename) if err != nil { return fmt.Errorf("unable to read file: %w", err) } @@ -298,7 +402,12 @@ func fixIdentifiers(fn string) error { data = bytes.ReplaceAll(data, []byte(cur), []byte(repl)) } - err = ioutil.WriteFile(fn, data, 0644) + // Inject the correct import path. + packageDir := path.Base(path.Dir(cfg.outputFilename)) + importPath := fmt.Sprintf(`"go.opentelemetry.io/otel/semconv/%s"`, packageDir) + data = bytes.ReplaceAll(data, []byte(`[[IMPORTPATH]]`), []byte(importPath)) + + err = ioutil.WriteFile(cfg.outputFilename, data, 0644) if err != nil { return fmt.Errorf("unable to write updated file: %w", err) } diff --git a/internal/tools/semconv-gen/template.j2 b/internal/tools/semconv-gen/template.j2 index 63095e3b1..1e94d1375 100644 --- a/internal/tools/semconv-gen/template.j2 +++ b/internal/tools/semconv-gen/template.j2 @@ -46,7 +46,7 @@ Note: {{ attr.note }} // Code generated from semantic convention specification. DO NOT EDIT. -package semconv // import "go.opentelemetry.io/otel/semconv" +package semconv // import [[IMPORTPATH]] import "go.opentelemetry.io/otel/attribute"