Merge #2748

2748: Improve ci by using intermediate images r=mergify[bot] a=Diman0 ## What type of PR? enhancement ## What does this PR do? As discussed in the mailu-dev chat, to improve build times we should 1. First build the base image and push this to ghcr.io 2. Secondly at the same time build the assets images and push this to ghcr.io. 3. Simultanuously build all other images which use the base/assets image from ghcr.io. I've made these changes to the build_test_deploy.yml file and build-ci.yml file. https://github.com/Mailu/Mailu/actions/runs/4637399505 In the worst case the build time appears to be around ~13 minutes now. - build-ci.hcl will be used only by the github workflow. This file is modified to pull the base/assets image from the registry. - build.hcl is unmodified. This file can be used to build all images locally. I changed the name of the build jobs to make sure they fit in the overview list when viewing the action on github. You can now actually identify what job is run for what matrix target. ### Related issue(s) ## Prerequisites Before we can consider review and merge, please make sure the following list is done and checked. If an entry in not applicable, you can check it or remove it from the list. - [n/a ] In case of feature or enhancement: documentation updated accordingly - [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file. Co-authored-by: Dimitri Huisman <diman@huisman.xyz>
2024-12-12 10:45:38 +02:00 · 2023-04-07 11:20:04 +00:00 · 2023-04-07 11:20:04 +00:00 · 60fc2b5679
commit 60fc2b5679
parent bcfe12a6f6 c6c4cd71f6
3 changed files with 410 additions and 9 deletions
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@ -80,7 +80,7 @@ on:
        required: false
        type: string
 env:
-  HCL_FILE: ./tests/build.hcl
+  HCL_FILE: ./tests/build-ci.hcl

 jobs:
 # This job calculates what images must be build. It reads the build.hcl file and then outputs all targets (images) in it.
@ -101,13 +101,162 @@ jobs:
        run: |
          echo ${{ steps.targets.outputs.matrix }}

-# This job builds all the images. The build cache is stored in the github actions cache.
-# In further jobs, this cache is used to quickly rebuild the images.
+# This job builds the base and assets images. The build cache is stored in the ghcr.io registry.
+  build-base-assets-x64:
+    name: x64 - build intermediate
+    if: contains(inputs.architecture, 'linux/amd64')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ["base", "assets"]
+    runs-on: ubuntu-latest
+    permissions:
+     contents: read
+     packages: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Retrieve global variables
+        shell: bash
+        run: |
+          echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV
+          echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
+          echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
+          echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - uses: crazy-max/ghaction-github-runtime@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ github.repository_owner }}
+      - name: Get uuid
+        id: uuid
+        run: |
+            echo uuid=$RANDOM >> $GITHUB_OUTPUT
+      - name: Build docker image with retry
+        env:
+          DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }}
+          MAILU_VERSION: ${{ env.MAILU_VERSION }}-build
+          PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}-build
+          LABEL_VERSION: ${{ env.MAILU_VERSION }}
+          PINNED_LABEL_VERSION: ${{ env.PINNED_MAILU_VERSION }}
+          ARCH: 'linux/amd64'
+          BUILDER: ${{ steps.uuid.outputs.uuid }}
+          DOCKER_LOGIN: ${{ secrets.Docker_Login }}
+          DOCKER_PASSW: ${{ secrets.Docker_Password }}
+          BUILDX_NO_DEFAULT_ATTESTATIONS: 1
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 20
+          retry_wait_seconds: 30
+          max_attempts: 3
+          shell: bash
+          command: |
+            set -euxo pipefail \
+              ; /usr/bin/docker info \
+              ; echo "${{ github.token }}" |        docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \
+              ; echo "$DOCKER_PASSW" |   docker login --username "$DOCKER_LOGIN" --password-stdin \
+              ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \
+              || echo "builder does not exist" \
+              ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \
+              ; /usr/bin/docker buildx bake --push \
+              --file ./tests/build-ci.hcl \
+              --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache \
+              --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max \
+              --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \
+              ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }}
+      - name: cleanup docker buildx instance after failure of build step
+        if: ${{ failure() }}
+        shell: bash
+        env:
+          BUILDER: ${{ steps.uuid.outputs.uuid }}
+        run: |
+          /usr/bin/docker buildx rm builder-${{ env.BUILDER }}
+
+# This job builds the base and assets images. The build cache is stored in the ghcr.io registry.
+  build-base-assets-arm:
+    name: ARM - build intermediate
+    if: contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ["base", "assets"]
+    runs-on: self-hosted
+    permissions:
+     contents: read
+     packages: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Retrieve global variables
+        shell: bash
+        run: |
+          echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV
+          echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
+          echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
+          echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - uses: crazy-max/ghaction-github-runtime@v2
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ github.repository_owner }}
+      #This is to prevent to shared runners from generating the same uuid
+      - name: Get unique random number
+        id: uuid
+        run: |
+            echo uuid=$RANDOM >> $GITHUB_OUTPUT
+      - name: Build docker image with retry
+        env:
+          DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }}
+          MAILU_VERSION: ${{ env.MAILU_VERSION }}-build-arm
+          PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}-build-arm
+          LABEL_VERSION: ${{ env.MAILU_VERSION }}
+          PINNED_LABEL_VERSION: ${{ env.PINNED_MAILU_VERSION }}
+          ARCH: linux/arm64/v8,linux/arm/v7
+          BUILDER: ${{ steps.uuid.outputs.uuid }}
+          DOCKER_LOGIN2: ${{ secrets.Docker_Login2 }}
+          DOCKER_PASSW2: ${{ secrets.Docker_Password2 }}
+          BUILDX_NO_DEFAULT_ATTESTATIONS: 1
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 30
+          retry_wait_seconds: 30
+          max_attempts: 10
+          shell: bash
+          command: |
+            set -euxo pipefail \
+              ; /usr/bin/docker info \
+              ; echo "${{ github.token }}" |        docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \
+              ; echo "$DOCKER_PASSW2" |    docker login --username "$DOCKER_LOGIN2" --password-stdin \
+              ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \
+              || echo "builder does not exist" \
+              ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \
+              ; /usr/bin/docker buildx bake --push \
+              --file ./tests/build-ci.hcl \
+              --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm \
+              --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max \
+              --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \
+              ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }}
+      - name: cleanup docker buildx instance after failure of build step
+        if: ${{ failure() }}
+        shell: bash
+        env:
+          BUILDER: ${{ steps.uuid.outputs.uuid }}
+        run: |
+          /usr/bin/docker buildx rm builder-${{ env.BUILDER }}
+
+# This job builds the main images. The build cache is stored in the ghcr.io registry.
  build:
-    name: Build images for linux/amd64
+    name: x64 - build
    if: contains(inputs.architecture, 'linux/amd64')
    needs:
      - targets
+      - build-base-assets-x64
    strategy:
      fail-fast: false
      matrix:
@ -166,7 +315,7 @@ jobs:
              || echo "builder does not exist" \
              ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \
              ; /usr/bin/docker buildx bake --push \
-              --file ./tests/build.hcl \
+              --file ./tests/build-ci.hcl \
              --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache \
              --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max \
              --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \
@ -179,13 +328,13 @@ jobs:
        run: |
          /usr/bin/docker buildx rm builder-${{ env.BUILDER }}

-# This job builds all the images. The build cache is stored in the github actions cache.
-# In further jobs, this cache is used to quickly rebuild the images.
+# This job builds the main images. The build cache is stored in the ghcr.io registry.
  build-arm:
-    name: Build images for ARM64 & ARM/V7
+    name: ARM - build
    if: contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7')
    needs:
      - targets
+      - build-base-assets-arm
    strategy:
      fail-fast: false
      matrix:
@ -243,7 +392,7 @@ jobs:
              || echo "builder does not exist" \
              ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \
              ; /usr/bin/docker buildx bake --push \
-              --file ./tests/build.hcl \
+              --file ./tests/build-ci.hcl \
              --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm \
              --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max \
              --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \
--- a/tests/build-ci.hcl
+++ b/tests/build-ci.hcl
@ -0,0 +1,250 @@
+# build-ci.hcl
+#
+# only used for Github actions workflow.
+# For locally building images use build.hcl
+#
+# For more information on buildx bake file definition see:
+# https://github.com/docker/buildx/blob/master/docs/guides/bake/file-definition.md
+#
+# NOTE: You can only run this from the Mailu root folder.
+# Make sure the context is Mailu (project folder) and not Mailu/tests
+#-----------------------------------------------------------------------------------------
+# (Environment) input variables
+# If the env var is not set, then the default value is used
+#-----------------------------------------------------------------------------------------
+variable "DOCKER_ORG" {
+  default = "ghcr.io/mailu"
+}
+variable "DOCKER_PREFIX" {
+  default = ""
+}
+variable "PINNED_MAILU_VERSION" {
+  default = "local"
+}
+variable "MAILU_VERSION" {
+  default = "local"
+}
+variable "LABEL_VERSION" {
+  default = "local"
+}
+variable "PINNED_LABEL_VERSION" {
+  default = "local"
+}
+
+#-----------------------------------------------------------------------------------------
+# Grouping of targets to build. All these images are built when using:
+# docker buildx bake -f tests\build.hcl
+#-----------------------------------------------------------------------------------------
+group "default" {
+  targets = [
+    "docs",
+    "setup",
+
+    "admin",
+    "antispam",
+    "front",
+    "imap",
+    "oletools",
+    "smtp",
+
+    "webmail",
+
+    "antivirus",
+    "fetchmail",
+    "resolver",
+    "traefik-certdumper",
+    "webdav"
+  ]
+}
+
+#-----------------------------------------------------------------------------------------
+# Default settings that will be inherited by all targets (images to build).
+#-----------------------------------------------------------------------------------------
+target "defaults" {
+  platforms = [ "linux/amd64"]
+  dockerfile = "Dockerfile"
+  args = {
+    VERSION = "${PINNED_LABEL_VERSION}"
+  }
+}
+
+#-----------------------------------------------------------------------------------------
+# User defined functions
+#------------------------------------------------------------------------------------------
+# Derive all tags
+function "tag" {
+  params = [image_name]
+  result = [  notequal("master",MAILU_VERSION) && notequal("master-arm",MAILU_VERSION) ? "${DOCKER_ORG}/${DOCKER_PREFIX}${image_name}:${PINNED_MAILU_VERSION}": "",
+             "${DOCKER_ORG}/${DOCKER_PREFIX}${image_name}:${MAILU_VERSION}",
+             "${DOCKER_ORG}/${DOCKER_PREFIX}${image_name}:latest"
+          ]
+}
+
+#-----------------------------------------------------------------------------------------
+# All individual targets (images to build)
+# Build an individual target using.
+# docker buildx bake -f tests\build.hcl <target>
+# E.g. to build target docs
+# docker buildx bake -f tests\build.hcl docs
+#-----------------------------------------------------------------------------------------
+
+# -----------------------------------------------------------------------------------------
+# Base images
+# -----------------------------------------------------------------------------------------
+target "base" {
+  inherits = ["defaults"]
+  context = "core/base/"
+  tags = tag("base")
+}
+
+target "assets" {
+  inherits = ["defaults"]
+  context = "core/admin/assets/"
+  tags = tag("assets")
+}
+
+# -----------------------------------------------------------------------------------------
+# Documentation and setup images
+# -----------------------------------------------------------------------------------------
+target "docs" {
+  inherits = ["defaults"]
+  context = "docs/"
+  tags = tag("docs")
+  args = {
+    version = "${LABEL_VERSION}"
+    pinned_version = "${PINNED_LABEL_VERSION}"
+  }
+}
+
+target "setup" {
+  inherits = ["defaults"]
+  context = "setup/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("setup")
+}
+
+# -----------------------------------------------------------------------------------------
+# Core images
+# -----------------------------------------------------------------------------------------
+target "none" {
+  inherits = ["defaults"]
+  context = "core/none/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("none")
+}
+
+target "admin" {
+  inherits = ["defaults"]
+  context = "core/admin/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+    assets = "docker-image://${DOCKER_ORG}/assets:${MAILU_VERSION}"
+  }
+  tags = tag("admin")
+}
+
+target "antispam" {
+  inherits = ["defaults"]
+  context = "core/rspamd/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("rspamd")
+}
+
+target "front" {
+  inherits = ["defaults"]
+  context = "core/nginx/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("nginx")
+}
+
+target "oletools" {
+  inherits = ["defaults"]
+  context = "core/oletools/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("oletools")
+}
+
+target "imap" {
+  inherits = ["defaults"]
+  context = "core/dovecot/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("dovecot")
+}
+
+target "smtp" {
+  inherits = ["defaults"]
+  context = "core/postfix/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("postfix")
+}
+
+# -----------------------------------------------------------------------------------------
+# Webmail image
+# -----------------------------------------------------------------------------------------
+target "webmail" {
+  inherits = ["defaults"]
+  context = "webmails/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("webmail")
+}
+
+# -----------------------------------------------------------------------------------------
+# Optional images
+# -----------------------------------------------------------------------------------------
+target "antivirus" {
+  inherits = ["defaults"]
+  context = "optional/clamav/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("clamav")
+}
+
+target "fetchmail" {
+  inherits = ["defaults"]
+  context = "optional/fetchmail/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("fetchmail")
+}
+
+target "resolver" {
+  inherits = ["defaults"]
+  context = "optional/unbound/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("unbound")
+}
+
+target "traefik-certdumper" {
+  inherits = ["defaults"]
+  context = "optional/traefik-certdumper/"
+  tags = tag("traefik-certdumper")
+}
+
+target "webdav" {
+  inherits = ["defaults"]
+  context = "optional/radicale/"
+  contexts = {
+    base = "docker-image://${DOCKER_ORG}/base:${MAILU_VERSION}"
+  }
+  tags = tag("radicale")
+}
--- a/towncrier/newsfragments/2748.misc
+++ b/towncrier/newsfragments/2748.misc
@ -0,0 +1,2 @@
+Improve CI/CD workflow to first build the intermediate images which are required for building the main images.
+This should drastically improve the time required for builds in the worst cache scenario (no cache hits).