From e413dfc153a1a567f231bbf1a7e9d9fbabec1237 Mon Sep 17 00:00:00 2001
From: Sam Xie <sam@samxie.me>
Date: Tue, 3 Mar 2026 10:42:54 -0800
Subject: [PATCH] Refactor benchmark CI (#7873)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Resolves #4537

This PR replaces the `github-action-benchmark` approach with CodSpeed.

CodSpeed addresses:
- The benchmark conflict issue:
https://github.com/open-telemetry/opentelemetry-go/issues/4537#issuecomment-2294034235
- The maintenance burden for GitHub pages.
- It allows partial performance test, which enables us to run benchmark
for PRs in the future

An example of benchmark result:
https://codspeed.io/XSAM/opentelemetry-go/runs/697aa351f5662999a88a8bf3

Benchmark page for opentelemetry-go:
https://codspeed.io/open-telemetry/opentelemetry-go

---------

Co-authored-by: Robert Pająk <pellared@hotmail.com>
---
 .github/workflows/benchmark.yml | 55 ++++++++++++++++-----------------
 Makefile                        |  9 +++---
 2 files changed, 30 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 314ec7bd0..533f8ac58 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -11,11 +11,27 @@ permissions: read-all
 env:
   DEFAULT_GO_VERSION: "~1.26.0"
 jobs:
+  # Related issue: https://github.com/CodSpeedHQ/codspeed-go/issues/51
+  sharding-benchmark:
+    name: Sharding benchmarks
+    runs-on: ubuntu-latest
+    outputs:
+      shards: ${{ steps.sharding.outputs.shards }}
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - id: sharding
+        run: |
+          echo "shards=$(make print-sharded-benchmarks)" >> $GITHUB_OUTPUT
+
   benchmark:
+    needs: sharding-benchmark
     permissions:
       contents: write # required for pushing to gh-pages branch
     name: Benchmarks
     runs-on: oracle-bare-metal-64cpu-512gb-x86-64
+    strategy:
+      matrix:
+        shard: ${{ fromJson(needs.sharding-benchmark.outputs.shards) }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
@@ -23,33 +39,14 @@ jobs:
           go-version: ${{ env.DEFAULT_GO_VERSION }}
           check-latest: true
           cache-dependency-path: "**/go.sum"
-      - name: Run benchmarks
-        run: make benchmark | tee output.txt
-      - name: Download previous benchmark data
-        uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v4.10.1
         with:
-          path: ./benchmarks
-          # `github.event.before` means the commit before the push (i.e. the previous commit).
-          # So we can fetch the exact benchmark data from the previous commit.
-          key: ${{ runner.os }}-benchmark-${{ github.event.before }}
-      - name: Store benchmarks result
-        uses: benchmark-action/github-action-benchmark@a7bc2366eda11037936ea57d811a43b3418d3073 # v1.21.0
-        with:
-          name: Benchmarks
-          tool: 'go'
-          output-file-path: output.txt
-          external-data-json-path: ./benchmarks/data.json
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          gh-pages-branch: benchmarks
-          fail-on-alert: true
-          alert-threshold: "400%"
-          # Add benchmark summary to GitHub workflow run report
-          summary-always: true
-      - name: Save benchmark data
-        uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
-        # The cache will be saved even if a step fails.
-        if: always()
-        with:
-          path: ./benchmarks
-          # Use the current commit SHA as the cache key.
-          key: ${{ runner.os }}-benchmark-${{ github.sha }}
+          mode: walltime
+          allow-empty: true
+          # CodSpeed overrides the default benchtime if we don't explicitly specify it.
+          # Having this would avoid running benchmark for more than 1 hour.
+          #
+          # The benchtime is adjusted to 500 ms to speed up the benchmark time.
+          # Per https://github.com/open-telemetry/community/issues/2331#issuecomment-2356403352
+          run: make benchmark/${{matrix.shard}} ARGS=-benchtime=500ms
diff --git a/Makefile b/Makefile
index fc4befb22..9f6e6b511 100644
--- a/Makefile
+++ b/Makefile
@@ -185,11 +185,10 @@ test-coverage: $(GOCOVMERGE)
 .PHONY: benchmark
 benchmark: $(OTEL_GO_MOD_DIRS:%=benchmark/%)
 benchmark/%:
-	@echo "$(GO) test -run=xxxxxMatchNothingxxxxx -bench=. $*..." \
-		&& cd $* \
-		&& $(GO) list ./... \
-		| grep -v third_party \
-		| xargs $(GO) test -run=xxxxxMatchNothingxxxxx -bench=.
+	cd $* && $(GO) test -run='^$$' -bench=. $(ARGS) ./...
+
+print-sharded-benchmarks:
+	@echo $(OTEL_GO_MOD_DIRS) | jq -cR 'split(" ")'
 
 .PHONY: golangci-lint golangci-lint-fix
 golangci-lint-fix: ARGS=--fix