(feat) Golang integration tests dynamic workflow

* Dynamic version detection of Golang 

* Dynamic Golang integration tests run matrix

* Golang timeout to build set at 5 min

* Golang integration test timeout to run set at 10 min

* Consumer tests timeout set at 20 min

Co-authored-by: Alexey Matvievsky <alexey.matvievsky@sap.com>

.github/workflows/consumer-tests-pr.yml

@@ -19,6 +19,7 @@ jobs:
         github.event.comment.author_association == 'MEMBER'
       )
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
       - uses: styfle/cancel-workflow-action@0.10.0
       - name: Get pull request URL

.github/workflows/consumer-tests.yml

@@ -10,6 +10,7 @@ jobs:
   consumer_tests:
     name: Consumer tests
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
       - uses: styfle/cancel-workflow-action@0.10.0
       - uses: actions/checkout@v3

.github/workflows/integration-tests-pr.yml

@@ -6,13 +6,6 @@ on:
       - created
       - edited
 
-env:
-  GHA_GO_VERSION: 1.18.x
-  TIMEOUT: 10m
-
-  PIPER_INTEGRATION_GITHUB_TOKEN: ${{secrets.PIPER_INTEGRATION_GITHUB_TOKEN}}
-  PIPER_INTEGRATION_SONAR_TOKEN: ${{secrets.PIPER_INTEGRATION_SONAR_TOKEN}}
-
 jobs:
   start:
     name: Start
@@ -26,6 +19,7 @@ jobs:
         github.event.comment.author_association == 'MEMBER'
       )
     outputs:
+      go_version: ${{ steps.go_version.outputs.go_version }}
       sha: ${{ steps.sha.outputs.sha }}
     runs-on: ubuntu-latest
     steps:
@@ -52,6 +46,10 @@ jobs:
         with:
           repository: ${{ steps.repository.outputs.repository }}
           ref: ${{ steps.branch_name.outputs.branch_name }}
+      - name: Get GO version
+        id: go_version
+        run: |
+          echo "::set-output name=go_version::$(cat go.mod | grep go | head -1 | awk '{print $2}')"
       - name: Get commit SHA
         id: sha
         run: |
@@ -78,7 +76,7 @@ jobs:
           ref: ${{ needs.start.outputs.sha }}
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GHA_GO_VERSION }}
+          go-version: ${{ needs.start.outputs.go_version }}
       - name: Build
         # with `-tags release` we ensure that shared test utilities won't end up in the binary
         run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o piper -tags release
@@ -91,6 +89,8 @@ jobs:
 
   build_integration_tests:
     name: Build integration tests
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
     needs:
       - start
     runs-on: ubuntu-latest
@@ -100,9 +100,14 @@ jobs:
           ref: ${{ needs.start.outputs.sha }}
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GHA_GO_VERSION }}
+          go-version: ${{ needs.start.outputs.go_version }}
       - name: Build
         run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o integration_tests -tags integration ./integration/...
+      - name: Form integration tests run matrix
+        if: success()
+        id: matrix
+        run: |
+          echo "::set-output name=matrix::$(go run .github/workflows/parse_integration_test_list.go -file ./integration/github_actions_integration_test_list.yml)"
       - name: Upload integration tests binary
         if: success()
         uses: actions/upload-artifact@v3
@@ -117,40 +122,16 @@ jobs:
       - build_integration_tests
     strategy:
       fail-fast: true
-      matrix:
+      matrix: ${{ fromJson(needs.build_integration_tests.outputs.matrix) }}
-        os:
+    runs-on: ubuntu-latest
-          - ubuntu-latest
+    timeout-minutes: 10
-        version:
-          # tough CNB tests are decomposed to reduce pipeline execution time
-          - '"(TestCNBIntegrationBindings|TestCNBIntegrationWrongBuilderProject|TestCNBIntegrationNonZipPath|TestCNBIntegrationZipPath|TestCNBIntegrationPreserveFilesIgnored)"'
-          - '"TestCNBIntegrationPreserveFiles\b"'
-          - '"TestCNBIntegrationNPMProject"'
-          - '"TestCNBIntegrationMultiImage"'
-          - '"TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject"'
-          - '"TestCNBIntegrationNPMCustomBuildpacksFullProject"'
-          - '"TestCNBIntegrationProjectDescriptor"'
-
-          - '"TestGolangIntegration"'
-          - '"TestGradleIntegration"'
-
-          # Jenkins tests are still not implemented(skipped) yet
-          # - '"TestJenkinsIntegration"'
-
-          - '"TestMavenIntegration"'
-          - '"TestMTAIntegration"'
-          - '"TestNexusIntegration"'
-
-          # these are light-weighted tests, so we can use only one pod to reduce resource consumption
-          - '"Test(Gauge|GCS|GitHub|GitOps|Influx|NPM|Piper|Python|Sonar|Vault|Karma)Integration"'
-
-    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ needs.start.outputs.sha }}
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GHA_GO_VERSION }}
+          go-version: ${{ needs.start.outputs.go_version }}
       - name: Download Piper binary
         uses: actions/download-artifact@v3
         with:
@@ -161,13 +142,16 @@ jobs:
           name: integration_tests
           path: ./integration/
       - name: Run test
+        env:
+          PIPER_INTEGRATION_GITHUB_TOKEN: ${{secrets.PIPER_INTEGRATION_GITHUB_TOKEN}}
+          PIPER_INTEGRATION_SONAR_TOKEN: ${{secrets.PIPER_INTEGRATION_SONAR_TOKEN}}
         run: |
           chmod +x piper
           cd ./integration
           chmod +x integration_tests
-          ./integration_tests -test.v -test.timeout ${TIMEOUT} -test.run ${{ matrix.version }}
+          ./integration_tests -test.v -test.run ${{ matrix.run }}
 
-  Finish:
+  finish:
     name: Finish
     if: always() && needs.start.result == 'success'
     needs:

.github/workflows/integration-tests.yml

@@ -6,22 +6,20 @@ on:
       - master
       - it/**
 
-env:
-  GHA_GO_VERSION: 1.18.x
-  TIMEOUT: 10m
-
-  PIPER_INTEGRATION_GITHUB_TOKEN: ${{secrets.PIPER_INTEGRATION_GITHUB_TOKEN}}
-  PIPER_INTEGRATION_SONAR_TOKEN: ${{secrets.PIPER_INTEGRATION_SONAR_TOKEN}}
-
 jobs:
   start:
     name: Start
     outputs:
+      go_version: ${{ steps.go_version.outputs.go_version }}
       sha: ${{ steps.sha.outputs.sha }}
     runs-on: ubuntu-latest
     steps:
       - uses: styfle/cancel-workflow-action@0.10.0
       - uses: actions/checkout@v3
+      - name: Get GO version
+        id: go_version
+        run: |
+          echo "::set-output name=go_version::$(cat go.mod | grep go | head -1 | awk '{print $2}')"
       - name: Get commit SHA
         id: sha
         run: |
@@ -42,13 +40,14 @@ jobs:
     needs:
       - start
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ needs.start.outputs.sha }}
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GHA_GO_VERSION }}
+          go-version: ${{ needs.start.outputs.go_version }}
       - name: Build
         # with `-tags release` we ensure that shared test utilities won't end up in the binary
         run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o piper -tags release
@@ -61,18 +60,26 @@ jobs:
 
   build_integration_tests:
     name: Build integration tests
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
     needs:
       - start
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ needs.start.outputs.sha }}
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GHA_GO_VERSION }}
+          go-version: ${{ needs.start.outputs.go_version }}
       - name: Build
         run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o integration_tests -tags integration ./integration/...
+      - name: Form integration tests run matrix
+        if: success()
+        id: matrix
+        run: |
+          echo "::set-output name=matrix::$(go run .github/workflows/parse_integration_test_list.go -file ./integration/github_actions_integration_test_list.yml)"
       - name: Upload integration tests binary
         if: success()
         uses: actions/upload-artifact@v3
@@ -87,40 +94,16 @@ jobs:
       - build_integration_tests
     strategy:
       fail-fast: true
-      matrix:
+      matrix: ${{ fromJson(needs.build_integration_tests.outputs.matrix) }}
-        os:
+    runs-on: ubuntu-latest
-          - ubuntu-latest
+    timeout-minutes: 10
-        version:
-          # tough CNB tests are decomposed to reduce pipeline execution time
-          - '"(TestCNBIntegrationBindings|TestCNBIntegrationWrongBuilderProject|TestCNBIntegrationNonZipPath|TestCNBIntegrationZipPath|TestCNBIntegrationPreserveFilesIgnored)"'
-          - '"TestCNBIntegrationPreserveFiles\b"'
-          - '"TestCNBIntegrationNPMProject"'
-          - '"TestCNBIntegrationMultiImage"'
-          - '"TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject"'
-          - '"TestCNBIntegrationNPMCustomBuildpacksFullProject"'
-          - '"TestCNBIntegrationProjectDescriptor"'
-
-          - '"TestGolangIntegration"'
-          - '"TestGradleIntegration"'
-
-          # Jenkins tests are still not implemented(skipped) yet
-          # - '"TestJenkinsIntegration"'
-
-          - '"TestMavenIntegration"'
-          - '"TestMTAIntegration"'
-          - '"TestNexusIntegration"'
-
-          # these are light-weighted tests, so we can use only one pod to reduce resource consumption
-          - '"Test(Gauge|GCS|GitHub|GitOps|Influx|NPM|Piper|Python|Sonar|Vault|Karma)Integration"'
-
-    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ needs.start.outputs.sha }}
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GHA_GO_VERSION }}
+          go-version: ${{ needs.start.outputs.go_version }}
       - name: Download Piper binary
         uses: actions/download-artifact@v3
         with:
@@ -131,13 +114,16 @@ jobs:
           name: integration_tests
           path: ./integration/
       - name: Run test
+        env:
+          PIPER_INTEGRATION_GITHUB_TOKEN: ${{secrets.PIPER_INTEGRATION_GITHUB_TOKEN}}
+          PIPER_INTEGRATION_SONAR_TOKEN: ${{secrets.PIPER_INTEGRATION_SONAR_TOKEN}}
         run: |
           chmod +x piper
           cd ./integration
           chmod +x integration_tests
-          ./integration_tests -test.v -test.timeout ${TIMEOUT} -test.run ${{ matrix.version }}
+          ./integration_tests -test.v -test.run ${{ matrix.run }}
 
-  Finish:
+  finish:
     name: Finish
     if: always() && needs.start.result == 'success'
     needs:

.github/workflows/parse_integration_test_list.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"github.com/ghodss/yaml"
+	"os"
+)
+
+func main() {
+	file := flag.String("file", "../../integration/github_actions_integration_test_list.yml", "Tests to be executed")
+	flag.Parse()
+	f, _ := os.ReadFile(*file)
+	var Matrix struct {
+		Run interface{} `json:"run,omitempty" yaml:"run"`
+	}
+	yaml.Unmarshal(f, &Matrix)
+	output, _ := json.Marshal(Matrix)
+	fmt.Println(string(output))
+}

integration/github_actions_integration_test_list.yml

@@ -0,0 +1,24 @@
+# Every run element will be executed in a different pod of job matrix
+# Use it to optimize resource consumption and execution time
+run:
+  # tough CNB tests are decomposed to reduce pipeline execution time
+  - '"(TestCNBIntegrationBindings|TestCNBIntegrationWrongBuilderProject|TestCNBIntegrationNonZipPath|TestCNBIntegrationZipPath|TestCNBIntegrationPreserveFilesIgnored)"'
+  - '"TestCNBIntegrationPreserveFiles\b"'
+  - '"TestCNBIntegrationNPMProject"'
+  - '"TestCNBIntegrationMultiImage"'
+  - '"TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject"'
+  - '"TestCNBIntegrationNPMCustomBuildpacksFullProject"'
+  - '"TestCNBIntegrationProjectDescriptor"'
+
+  - '"TestGolangIntegration"'
+  - '"TestGradleIntegration"'
+
+  # Jenkins tests are still not implemented(skipped) yet
+  #- '"TestJenkinsIntegration"'
+
+  - '"TestMavenIntegration"'
+  - '"TestMTAIntegration"'
+  - '"TestNexusIntegration"'
+
+  # these are light-weighted tests, so we can use only one pod to reduce resource consumption
+  - '"Test(Gauge|GCS|GitHub|GitOps|Influx|NPM|Piper|Python|Sonar|Vault|Karma)Integration"'