From 30dfdb4072c80da2a2bd30524c2b00b80f37b943 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Mon, 27 Mar 2023 13:21:12 +0000 Subject: [PATCH] Make sure that the arm build also uses build-cache. Remove the step of building the base image. This is not required. when it is build for the first time for an image, it will be part of the build cache of that image. --- .github/workflows/build_test_deploy.yml | 151 ++---------------------- 1 file changed, 10 insertions(+), 141 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 9ec997b6..7a9b9371 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -101,143 +101,6 @@ jobs: run: | echo ${{ steps.targets.outputs.matrix }} -## This job builds the base image. The base image is used by all other images. - build-base-image-x64: - name: Build base image x64 - if: contains(inputs.architecture, 'linux/amd64') - needs: - - targets - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@v3 - - name: Retrieve global variables - shell: bash - run: | - echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV - echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - uses: crazy-max/ghaction-github-runtime@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Helper to convert docker org to lowercase - id: string - uses: ASzc/change-string-case-action@v5 - with: - string: ${{ github.repository_owner }} - - name: Get uuid - id: uuid - run: | - echo uuid=$RANDOM >> $GITHUB_OUTPUT - - name: Build docker base image with retry - env: - DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - LABEL_VERSION: ${{ env.MAILU_VERSION }} - PINNED_LABEL_VERSION: ${{ env.PINNED_MAILU_VERSION }} - ARCH: 'linux/amd64' - BUILDER: ${{ steps.uuid.outputs.uuid }} - DOCKER_LOGIN: ${{ secrets.Docker_Login }} - DOCKER_PASSW: ${{ secrets.Docker_Password }} - BUILDX_NO_DEFAULT_ATTESTATIONS: 1 - uses: nick-fields/retry@v2 - with: - timeout_minutes: 20 - retry_wait_seconds: 30 - max_attempts: 3 - shell: bash - command: | - set -euxo pipefail \ - ; /usr/bin/docker info \ - ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ - ; echo "$DOCKER_PASSW" | docker login --username "$DOCKER_LOGIN" --password-stdin \ - ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ - || echo "builder does not exist" \ - ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \ - ; /usr/bin/docker buildx bake --file ./tests/build.hcl --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }},mode=max --set *.platform=${{ env.ARCH }} base \ - ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} - - name: cleanup docker buildx instance after failure of build step - if: ${{ failure() }} - shell: bash - env: - BUILDER: ${{ steps.uuid.outputs.uuid }} - run: | - /usr/bin/docker buildx rm builder-${{ env.BUILDER }} - -## This job builds the base image. The base image is used by all other images. - build-base-image-arm: - name: Build base image arm - if: contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7') - needs: - - targets - runs-on: self-hosted - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@v3 - - name: Retrieve global variables - shell: bash - run: | - echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV - echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - uses: crazy-max/ghaction-github-runtime@v2 - - name: Helper to convert docker org to lowercase - id: string - uses: ASzc/change-string-case-action@v5 - with: - string: ${{ github.repository_owner }} - - name: Get uuid - id: uuid - run: | - echo uuid=$RANDOM >> $GITHUB_OUTPUT - - name: Build docker base image with retry - env: - DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} - MAILU_VERSION: ${{ env.MAILU_VERSION }}-arm - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}-arm - LABEL_VERSION: ${{ env.MAILU_VERSION }} - PINNED_LABEL_VERSION: ${{ env.PINNED_MAILU_VERSION }} - ARCH: linux/arm64/v8,linux/arm/v7 - BUILDER: ${{ steps.uuid.outputs.uuid }} - DOCKER_LOGIN2: ${{ secrets.Docker_Login2 }} - DOCKER_PASSW2: ${{ secrets.Docker_Password2 }} - BUILDX_NO_DEFAULT_ATTESTATIONS: 1 - uses: nick-fields/retry@v2 - with: - timeout_minutes: 30 - retry_wait_seconds: 30 - max_attempts: 10 - shell: bash - command: | - set -euxo pipefail \ - ; /usr/bin/docker info \ - ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ - ; echo "$DOCKER_PASSW2" | docker login --username "$DOCKER_LOGIN2" --password-stdin \ - ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ - || echo "builder does not exist" \ - ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \ - ; /usr/bin/docker buildx bake --file ./tests/build.hcl --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max --set *.platform=${{ env.ARCH }} base \ - ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} - - name: cleanup docker buildx instance after failure of build step - if: ${{ failure() }} - shell: bash - env: - BUILDER: ${{ steps.uuid.outputs.uuid }} - run: | - /usr/bin/docker buildx rm builder-${{ env.BUILDER }} - - # This job builds all the images. The build cache is stored in the github actions cache. # In further jobs, this cache is used to quickly rebuild the images. build: @@ -245,7 +108,6 @@ jobs: if: contains(inputs.architecture, 'linux/amd64') needs: - targets - - build-base-image-x64 strategy: fail-fast: false matrix: @@ -303,7 +165,11 @@ jobs: ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ || echo "builder does not exist" \ ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \ - ; /usr/bin/docker buildx bake --push --file ./tests/build.hcl --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \ + ; /usr/bin/docker buildx bake --push \ + --file ./tests/build.hcl \ + --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache \ + --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max \ + --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \ ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} - name: cleanup docker buildx instance after failure of build step if: ${{ failure() }} @@ -320,7 +186,6 @@ jobs: if: contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7') needs: - targets - - build-base-image-arm strategy: fail-fast: false matrix: @@ -377,7 +242,11 @@ jobs: ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ || echo "builder does not exist" \ ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \ - ; /usr/bin/docker buildx bake --push --file ./tests/build.hcl --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \ + ; /usr/bin/docker buildx bake --push \ + --file ./tests/build.hcl \ + --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm \ + --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max \ + --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \ ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} - name: cleanup docker buildx instance after failure of build step if: ${{ failure() }}