Add compatibility tests to CI (#1567)

* Add compatibility tests to CI * Increase timeout to 60s Some of the aggregator tests are running just over 30 seconds when testing on Windows. Bump this to allow for it. * Remove cache setup jobs Having a unified job setup the cache when none exists so packages are only downloaded once is ideal, but these steps are taking too long. In some situations they are doubling the run time. Remove them and prefer to have multiple downloads when dependencies change. * Add changes to changelog * Remove mod-download target from Makefile Unused now that the cache jobs are removed. * Add compatibility testing table to changelog * Add change about precommit to changelog * Update README * Correct examples target in lint job
2024-12-26 21:05:00 +02:00 · 2021-03-01 08:38:23 -08:00 · 2021-03-01 08:38:23 -08:00 · e25503a00e
commit e25503a00e
parent 1519d95982
4 changed files with 275 additions and 147 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -5,46 +5,142 @@ on:
      - main
  pull_request:
 env:
-  TEST_RESULTS: /tmp/test-results # path to where test results will be saved
+  # Path to where test results will be saved.
+  TEST_RESULTS: /tmp/test-results
+  # Default minimum version of Go to support.
+  DEFAULT_GO_VERSION: 1.14
 jobs:
-  go:
-    strategy:
-      fail-fast: false
-      matrix:
-        go_version: [1.14, 1.15]
+  lint:
    runs-on: ubuntu-latest
    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v2
-      - name: Setup Go
-        uses: actions/setup-go@v2.1.3
-        with:
-          go-version: ${{ matrix.go_version }}
-      - name: Setup Environment
-        run: |
-          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
-          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
-      - name: restore cache
-        uses: actions/cache@v2.1.4
-        env:
-          cache-name: cache-go-mod
-        with:
-          path: /home/runner/go/pkg/mod
-          key: go-pkg-mod-{{ checksum "go.sum" }}
-      - name: Precommit and Coverage Report
-        run: |
-          make ci
-          mkdir $TEST_RESULTS
-          cp coverage.out $TEST_RESULTS
-          cp coverage.txt $TEST_RESULTS
-          cp coverage.html $TEST_RESULTS
-      - uses: codecov/codecov-action@v1
-        with:
-          file: ./coverage.txt
-          fail_ci_if_error: true
-          verbose: true
-      - name: store test output
-        uses: actions/upload-artifact@v2
-        with:
-            name: opentelemetry-go-test-output
-            path: ${{ env.TEST_RESULTS }}
+    - name: Install Go
+      uses: actions/setup-go@v2.1.3
+      with:
+        go-version: ${{ env.DEFAULT_GO_VERSION }}
+    - name: Checkout Repo
+      uses: actions/checkout@v2
+    - name: Setup Environment
+      run: |
+        echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+        echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+    - name: Module cache
+      uses: actions/cache@v2.1.4
+      env:
+        cache-name: go-mod-cache
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }}
+    - name: Tools cache
+      uses: actions/cache@v2.1.4
+      env:
+        cache-name: go-tools-cache
+      with:
+        path: ~/.tools
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('./internal/tools/**') }}
+    - name: Run linters
+      run: make dependabot-check license-check lint
+    - name: Build
+      run: make examples build
+    - name: Check clean repository
+      run: make check-clean-work-tree
+
+  test-race:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2.1.3
+      with:
+        go-version: ${{ env.DEFAULT_GO_VERSION }}
+    - name: Checkout Repo
+      uses: actions/checkout@v2
+    - name: Setup Environment
+      run: |
+        echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+        echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+    - name: Module cache
+      uses: actions/cache@v2.1.4
+      env:
+        cache-name: go-mod-cache
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }}
+    - name: Run tests with race detector
+      run: make test-race
+
+  test-coverage:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2.1.3
+      with:
+        go-version: ${{ env.DEFAULT_GO_VERSION }}
+    - name: Checkout Repo
+      uses: actions/checkout@v2
+    - name: Setup Environment
+      run: |
+        echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+        echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+    - name: Module cache
+      uses: actions/cache@v2.1.4
+      env:
+        cache-name: go-mod-cache
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }}
+    - name: Run coverage tests
+      run: |
+        make test-coverage
+        mkdir $TEST_RESULTS
+        cp coverage.out $TEST_RESULTS
+        cp coverage.txt $TEST_RESULTS
+        cp coverage.html $TEST_RESULTS
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.txt
+        fail_ci_if_error: true
+        verbose: true
+    - name: Store coverage test output
+      uses: actions/upload-artifact@v2
+      with:
+          name: opentelemetry-go-test-output
+          path: ${{ env.TEST_RESULTS }}
+
+  compatibility-test:
+    strategy:
+      matrix:
+        go-version: [1.15, 1.14]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        # GitHub Actions does not support arm* architectures on default
+        # runners. It is possible to acomplish this with a self-hosted runner
+        # if we want to add this in the future:
+        # https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow
+        arch: ["386", amd64]
+        exclude:
+        # Not a supported Go OS/architecture.
+        - os: macos-latest
+          arch: "386"
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2.1.3
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Setup Environment
+      run: |
+        echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+        echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+      shell: bash
+    - name: Module cache
+      uses: actions/cache@v2.1.4
+      env:
+        cache-name: go-mod-cache
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }}
+    - name: Run tests
+      env:
+        GOARCH: ${{ matrix.arch }}
+      run: make test-short
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -14,6 +14,19 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - AttributePerEventCountLimit and AttributePerLinkCountLimit for SpanLimits. (#1535)
 - Added `Keys()` method to `propagation.TextMapCarrier` and `propagation.HeaderCarrier` to adapt `http.Header` to this interface. (#1544)
 - Added `code` attributes to `go.opentelemetry.io/otel/semconv` package. (#1558)
+- Compatibility testing suite in the CI system for the following systems. (#1567)
+   | OS      | Go Version | Architecture |
+   | ------- | ---------- | ------------ |
+   | Ubuntu  | 1.15       | amd64        |
+   | Ubuntu  | 1.14       | amd64        |
+   | Ubuntu  | 1.15       | 386          |
+   | Ubuntu  | 1.14       | 386          |
+   | MacOS   | 1.15       | amd64        |
+   | MacOS   | 1.14       | amd64        |
+   | Windows | 1.15       | amd64        |
+   | Windows | 1.14       | amd64        |
+   | Windows | 1.15       | 386          |
+   | Windows | 1.14       | 386          |
 - Changed all examples to use `WithBatchTimeout(5 * time.Second)` rather than `WithBatchTimeout(5)` (#1621)

 ### Changed
@ -24,6 +37,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Rename MaxEventsPerSpan, MaxAttributesPerSpan and MaxLinksPerSpan to EventCountLimit, AttributeCountLimit and LinkCountLimit, and move these fieds into SpanLimits. (#1535)
 - Renamed the `otel/label` package to `otel/attribute`. (#1541)
 - Vendor the Jaeger exporter's dependency on Apache Thrift. (#1551)
+- Parallelize the CI linting and testing. (#1567)
 - Stagger timestamps in exact aggregator tests. (#1569)
 - Prevent end-users from implementing some interfaces (#1575)
 ```
@ -44,6 +58,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 ### Removed

 - Removed attempt to resample spans upon changing the span name with `span.SetName()`. (#1545)
+- The `test-benchmark` is no longer a dependency of the `precommit` make target. (#1567)
+- The `test-386` make target.
+   This was replaced with a full compatibility testing suite (i.e. multi OS/arch) in the CI system. (#1567)

 ### Fixed

--- a/209
+++ b/209
@ -21,142 +21,127 @@ ALL_DOCS := $(shell find . -name '*.md' -type f | sort)
 ALL_GO_MOD_DIRS := $(filter-out $(TOOLS_MOD_DIR), $(shell find . -type f -name 'go.mod' -exec dirname {} \; | egrep -v '^./example' | sort)) $(shell find ./example -type f -name 'go.mod' -exec dirname {} \; | sort)
 ALL_COVERAGE_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | egrep -v '^./example|^$(TOOLS_MOD_DIR)' | sort)

-# Mac OS Catalina 10.5.x doesn't support 386. Hence skip 386 test
-SKIP_386_TEST = false
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S),Darwin)
-	SW_VERS := $(shell sw_vers -productVersion)
-	ifeq ($(shell echo $(SW_VERS) | egrep '^(10.1[5-9]|1[1-9]|[2-9])'), $(SW_VERS))
-		SKIP_386_TEST = true
-	endif
-endif
-
-GOTEST_MIN = go test -timeout 30s
-GOTEST = $(GOTEST_MIN) -race
-GOTEST_WITH_COVERAGE = $(GOTEST) -coverprofile=coverage.out -covermode=atomic -coverpkg=./...
+GO = go
+TIMEOUT = 60

 .DEFAULT_GOAL := precommit

-.PHONY: precommit
+.PHONY: precommit ci
+precommit: dependabot-check license-check lint build examples test-default
+ci: precommit check-clean-work-tree test-with-coverage

-TOOLS_DIR := $(abspath ./.tools)
+# Tools

-$(TOOLS_DIR)/golangci-lint: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go
+TOOLS = $(CURDIR)/.tools
+
+$(TOOLS):
+	@mkdir -p $@
+$(TOOLS)/%: | $(TOOLS)
 	cd $(TOOLS_MOD_DIR) && \
-	go build -o $(TOOLS_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint
+	$(GO) build -o $@ $(PACKAGE)

-$(TOOLS_DIR)/misspell: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go
-	cd $(TOOLS_MOD_DIR) && \
-	go build -o $(TOOLS_DIR)/misspell github.com/client9/misspell/cmd/misspell
+CROSSLINK = $(TOOLS)/crosslink
+$(TOOLS)/crosslink: PACKAGE=go.opentelemetry.io/otel/$(TOOLS_MOD_DIR)/crosslink

-$(TOOLS_DIR)/stringer: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go
-	cd $(TOOLS_MOD_DIR) && \
-	go build -o $(TOOLS_DIR)/stringer golang.org/x/tools/cmd/stringer
+GOLANGCI_LINT = $(TOOLS)/golangci-lint
+$(TOOLS)/golangci-lint: PACKAGE=github.com/golangci/golangci-lint/cmd/golangci-lint

-$(TOOLS_DIR)/gojq: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go
-	cd $(TOOLS_MOD_DIR) && \
-	go build -o $(TOOLS_DIR)/gojq github.com/itchyny/gojq/cmd/gojq
+MISSPELL = $(TOOLS)/misspell
+$(TOOLS)/misspell: PACKAGE= github.com/client9/misspell/cmd/misspell

-precommit: dependabot-check license-check generate build lint examples test-benchmarks test
+STRINGER = $(TOOLS)/stringer
+$(TOOLS)/stringer: PACKAGE=golang.org/x/tools/cmd/stringer

-.PHONY: test-with-coverage
-test-with-coverage:
-	set -e; \
+$(TOOLS)/gojq: PACKAGE=github.com/itchyny/gojq/cmd/gojq
+
+.PHONY: tools
+tools: $(CROSSLINK) $(GOLANGCI_LINT) $(MISSPELL) $(STRINGER) $(TOOLS)/gojq
+
+
+# Build
+
+.PHONY: examples generate build
+examples:
+	@set -e; for ex in $(EXAMPLES); do \
+	  echo "$(GO) build $${dir}/..."; \
+	  (cd "$${ex}" && \
+	   $(GO) build .); \
+	done
+
+generate: $(STRINGER)
+	set -e; for dir in $(ALL_GO_MOD_DIRS); do \
+	  echo "$(GO) generate $${dir}/..."; \
+	  (cd "$${dir}" && \
+	    PATH="$(TOOLS):$${PATH}" $(GO) generate ./...); \
+	done
+
+build: generate
+	# Build all package code including testing code.
+	set -e; for dir in $(ALL_GO_MOD_DIRS); do \
+	  echo "$(GO) build $${dir}/..."; \
+	  (cd "$${dir}" && \
+	    $(GO) build ./... && \
+		$(GO) list ./... \
+		  | grep -v third_party \
+		  | xargs $(GO) test -vet=off -run xxxxxMatchNothingxxxxx >/dev/null); \
+	done
+
+# Tests
+
+TEST_TARGETS := test-default test-bench test-short test-verbose test-race
+.PHONY: $(TEST_TARGETS) test
+test-default: ARGS=-v -race
+test-bench:   ARGS=-run=xxxxxMatchNothingxxxxx -test.benchtime=1ms -bench=.
+test-short:   ARGS=-short
+test-verbose: ARGS=-v
+test-race:    ARGS=-race
+$(TEST_TARGETS): test
+test:
+	@set -e; for dir in $(ALL_GO_MOD_DIRS); do \
+	  echo "$(GO) test -timeout $(TIMEOUT)s $(ARGS) $${dir}/..."; \
+	  (cd "$${dir}" && \
+	    $(GO) list ./... \
+		  | grep -v third_party \
+		  | xargs $(GO) test -timeout $(TIMEOUT)s $(ARGS)); \
+	done
+
+COVERAGE_MODE    = atomic
+COVERAGE_PROFILE = coverage.out
+.PHONY: test-coverage
+test-coverage:
+	@set -e; \
 	printf "" > coverage.txt; \
 	for dir in $(ALL_COVERAGE_MOD_DIRS); do \
-	  echo "go test ./... + coverage in $${dir}"; \
+	  echo "$(GO) test -coverpkg=./... -covermode=$(COVERAGE_MODE) -coverprofile="$(COVERAGE_PROFILE)" $${dir}/..."; \
 	  (cd "$${dir}" && \
-	 	go list ./... | grep -v third_party | xargs $(GOTEST_WITH_COVERAGE) && \
-		go tool cover -html=coverage.out -o coverage.html); \
+	 	$(GO) test -coverpkg=./... -covermode=$(COVERAGE_MODE) -coverprofile="$(COVERAGE_PROFILE)" ./... && \
+		$(GO) tool cover -html=coverage.out -o coverage.html); \
      [ -f "$${dir}/coverage.out" ] && cat "$${dir}/coverage.out" >> coverage.txt; \
 	done; \
 	sed -i.bak -e '2,$$ { /^mode: /d; }' coverage.txt

-
-.PHONY: ci
-ci: precommit check-clean-work-tree test-with-coverage test-386
-
-.PHONY: check-clean-work-tree
-check-clean-work-tree:
-	@if ! git diff --quiet; then \
-	  echo; \
-	  echo 'Working tree is not clean, did you forget to run "make precommit"?'; \
-	  echo; \
-	  git status; \
-	  exit 1; \
-	fi
-
-.PHONY: build
-build:
-	# TODO: Fix this on windows.
-	set -e; for dir in $(ALL_GO_MOD_DIRS); do \
-	  echo "compiling all packages in $${dir}"; \
-	  (cd "$${dir}" && \
-	    go build ./... && \
-	    go list ./... | grep -v third_party | xargs go test -vet=off -run xxxxxMatchNothingxxxxx >/dev/null); \
-	done
-
-.PHONY: test
-test:
-	set -e; for dir in $(ALL_GO_MOD_DIRS); do \
-	  echo "go test ./... + race in $${dir}"; \
-	  (cd "$${dir}" && \
-	    go list ./... | grep -v third_party | xargs $(GOTEST)); \
-	done
-
-.PHONY: test-386
-test-386:
-	if [ $(SKIP_386_TEST) = true ] ; then \
-	  echo "skipping the test for GOARCH 386 as it is not supported on the current OS"; \
-	else \
-	  set -e; for dir in $(ALL_GO_MOD_DIRS); do \
-	    echo "go test ./... GOARCH 386 in $${dir}"; \
-	    (cd "$${dir}" && \
-	      GOARCH=386 go list ./... | grep -v third_party | xargs $(GOTEST_MIN)); \
-	  done; \
-	fi
-
-.PHONY: examples
-examples:
-	@set -e; for ex in $(EXAMPLES); do \
-	  echo "Building $${ex}"; \
-	  (cd "$${ex}" && \
-	   go build .); \
-	done
-
-.PHONY: test-benchmarks
-test-benchmarks:
-	@set -e; for dir in $(ALL_GO_MOD_DIRS); do \
-	  echo "test benchmarks in $${dir}"; \
-	  (cd "$${dir}" && go list ./... | grep -v third_party | xargs go test -test.benchtime=1ms -run=NONE -bench=.) > /dev/null; \
-	done
-
 .PHONY: lint
-lint: $(TOOLS_DIR)/golangci-lint $(TOOLS_DIR)/misspell lint-modules
+lint: misspell lint-modules | $(GOLANGCI_LINT)
 	set -e; for dir in $(ALL_GO_MOD_DIRS); do \
 	  echo "golangci-lint in $${dir}"; \
 	  (cd "$${dir}" && \
-	    $(TOOLS_DIR)/golangci-lint run --skip-dirs-use-default --fix && \
-	    $(TOOLS_DIR)/golangci-lint run --skip-dirs-use-default); \
+	    $(GOLANGCI_LINT) run --fix && \
+	    $(GOLANGCI_LINT) run); \
 	done
-	$(TOOLS_DIR)/misspell -w $(ALL_DOCS)
+
+.PHONY: misspell
+misspell: | $(MISSPELL)
+	$(MISSPELL) -w $(ALL_DOCS)

 .PHONY: lint-modules
-lint-modules:
+lint-modules: | $(CROSSLINK)
 	set -e; for dir in $(ALL_GO_MOD_DIRS) $(TOOLS_MOD_DIR); do \
-	  echo "go mod tidy in $${dir}"; \
+	  echo "$(GO) mod tidy in $${dir}"; \
 	  (cd "$${dir}" && \
-	    go mod tidy); \
+	    $(GO) mod tidy); \
 	done
 	echo "cross-linking all go modules"
-	(cd internal/tools; go run ./crosslink)
-
-generate: $(TOOLS_DIR)/stringer
-	set -e; for dir in $(ALL_GO_MOD_DIRS); do \
-	  echo "running generators in $${dir}"; \
-	  (cd "$${dir}" && \
-	    PATH="$(TOOLS_DIR):$${PATH}" go generate ./...); \
-	done
+	$(CROSSLINK)

 .PHONY: license-check
 license-check:
@ -180,3 +165,13 @@ dependabot-check:
 		echo "missing go.mod dependabot check:"; echo "$$result"; \
 		exit 1; \
 	fi
+
+.PHONY: check-clean-work-tree
+check-clean-work-tree:
+	@if ! git diff --quiet; then \
+	  echo; \
+	  echo 'Working tree is not clean, did you forget to run "make precommit"?'; \
+	  echo; \
+	  git status; \
+	  exit 1; \
+	fi
--- a/README.md
+++ b/README.md
@ -27,6 +27,26 @@ and
 Project versioning information and stability guarantees can be found in the
 [versioning documentation](./VERSIONING.md).

+### Compatibility
+
+This project is tested on the following systems.
+
+| OS      | Go Version | Architecture |
+| ------- | ---------- | ------------ |
+| Ubuntu  | 1.15       | amd64        |
+| Ubuntu  | 1.14       | amd64        |
+| Ubuntu  | 1.15       | 386          |
+| Ubuntu  | 1.14       | 386          |
+| MacOS   | 1.15       | amd64        |
+| MacOS   | 1.14       | amd64        |
+| Windows | 1.15       | amd64        |
+| Windows | 1.14       | amd64        |
+| Windows | 1.15       | 386          |
+| Windows | 1.14       | 386          |
+
+While this project should work for other systems, no compatibility guarantees
+are made for those systems currently.
+
 ## Getting Started

 You can find a getting started guide on [opentelemetry.io](https://opentelemetry.io/docs/go/getting-started/).