feat: add buildkit component

.gitlab-ci.yml

@@ -25,6 +25,10 @@ include:
       context: $CI_PROJECT_DIR/tests
       dockerfile: $CI_PROJECT_DIR/tests/Dockerfile
       add_image_arch_postfix: true
+  - component: $CI_SERVER_HOST/$CI_PROJECT_PATH/buildkit-build@$CI_COMMIT_SHA
+    inputs:
+      context: $CI_PROJECT_DIR/tests
+      dockerfile: $CI_PROJECT_DIR/tests/Dockerfile
   - component: $CI_SERVER_HOST/$CI_PROJECT_PATH/buildah-build@$CI_COMMIT_SHA
     inputs:
       context: $CI_PROJECT_DIR/tests
@@ -46,12 +50,9 @@ include:
 
 # The above is all you need. But for testing the pipelines are run for amd64 and arm64.
 # So we override the jobs to use a matrix build. Again not needed by default.
-# Currently there is no small arm runner
-
 variables:
   GL_K8S_CONTEXT: gitlab-ci
   GL_K8S_AGENT_ID: 1103496
-
 pre-commit:
   tags: [$RUNNER]
   parallel:
@@ -69,10 +70,11 @@ snippets:
   parallel:
     matrix:
       - RUNNER: saas-linux-small-amd64
-        IMAGE: ["debian:bookworm-slim", "alpine"]
+        IMAGE: ["debian:trixie-slim", "alpine"]
       - RUNNER: saas-linux-small-arm64
-        IMAGE: ["debian:bookworm-slim", "alpine"]
+        IMAGE: ["debian:trixie-slim", "alpine"]
   script:
+    - echo "Running all snippets on ${CI_RUNNER_EXECUTABLE_ARCH}..."
     - !reference [.snippets, debian-core-tools]
     - !reference [.snippets, alpine-core-tools]
     - !reference [.snippets, openssl-install]
@@ -125,6 +127,15 @@ KANIKO:build:
       - RUNNER: saas-linux-small-amd64
       - RUNNER: saas-linux-small-arm64
 
+BUILDKIT:build:
+  tags: [$RUNNER]
+  # Multiple pushes may cause race condition. Retry twice
+  retry: 2
+  parallel:
+    matrix:
+      - RUNNER: saas-linux-small-amd64
+      - RUNNER: saas-linux-small-arm64
+
 TRIVY:container-scan:
   tags: [$RUNNER]
   allow_failure: true

templates/buildkit-build.md

@@ -0,0 +1,49 @@
+# Container Build: Buildkit
+
+### Usage
+
+Use this component to build container images in your project without the need of a privileged Docker runner. It uses the [buildkit](https://crazymax.dev/buildkit/) project to allow safe image builds in Docker or Kubernetes. It automatically adds metadata labels to the image for easy tracking when the image is deployed.
+You should add this component to an existing `.gitlab-ci.yml` file by using the `include:`
+keyword.
+
+```yaml
+include:
+  - component: gitlab.uni-assist.de/abn-dev/gitlab-ci-templates/buildkit-build@<VERSION>
+```
+
+where `<VERSION>` is the latest released tag or `main`. This will add a `container_build` job to the pipeline.  
+*NOTE:* By default the latest version of buildkit is used. For a more predictable outcome you should pin the version to a specific tag via the `buildctl_image` input.
+
+
+The template should work without modifications but you can customize the template settings.
+### Inputs
+
+| Input               | Default value                     | Description                                                                                                                  |
+| ------------------- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| `as_job`            | `BUILDKIT:build`                  | The name of the job that gets imported. Use ".my_job" to include as template.                                                |
+| `stage`             | `build`                           | The stage where you want the job to be added.                                                                                |
+| `buildctl_image`    | `moby/buildkit:latest`            | The Docker image of kaniko                                                                                                   |
+| `build_platform`    | `linux/amd64`                     | The The target platform to build for.                                                                                        |
+| `push`              | `true`                            | When set to `true` the image will be pushed to the default registry. Set to `false` to only build without pushing the image. |
+| `cache`             | `false`                           | When set to `true` cache layers will be saved to disk. Currently NOOP.                                                       |
+| `image_tags`        | ["$CI_COMMIT_REF_SLUG", "latest"] | Array of the image tags to build.                                                                                            |
+| `context`           | `$CI_PROJECT_DIR`                 | The build context.                                                                                                           |
+| `dockerfile`        | `$CI_PROJECT_DIR/Dockerfile`      | The path to the Dockerfile.                                                                                                  |
+| `authors`           | `$CI_COMMIT_AUTHOR`               | For OCI image label: Name of the image author(s).                                                                            |
+| `source_url`        | `$CI_PROJECT_URL`                 | For OCI image label: Url of the source code.                                                                                 |
+| `project_url`       | `$CI_PROJECT_URL`                 | For OCI image label: Url of the project.                                                                                     |
+| `documentation_url` | `$CI_PROJECT_URL`                 | For OCI image label: Url of the documentation.                                                                               |
+| `version`           | `$CI_COMMIT_REF_NAME`             | For OCI image label: Version of the image.                                                                                   |
+| `description`       | `$CI_PROJECT_DESCRIPTION`         | For OCI image label: Description.                                                                                            |
+| `vendor`            | `UNKNOWN`                         | For OCI image label: Vendor name.                                                                                            |
+| `license`           | `UNKNOWN`                         | For OCI image label: License.                                                                                                |
+| `services`          | buildkitd                         | The buildkitd service that actaually builds the image                                                                        |
+| `rules`             | *Default MR rules + Tags*         | The rules when the job runs                                                                                                  |
+
+### Variables
+
+| Variable                     | Description                                 |
+| ---------------------------- | ------------------------------------------- |
+| `GIT_STRATEGY`               | Default to `fetch`.                         |
+| `DOCKER_CONFIG_JSON`         | The registry auth json in the docker format |
+| `CONTAINER_BUILD_EXTRA_ARGS` | Extra args for the build engine.            |

templates/buildkit-build.yml

@@ -0,0 +1,184 @@
+spec:
+  inputs:
+    as_job:
+      type: string
+      default: "BUILDKIT:build"
+      description: The name of the job that will be included with this component.
+    stage:
+      type: string
+      default: "build"
+      description: The stage in which the job should run.
+    buildctl_image:
+      type: string
+      default: "moby/buildkit:latest"
+      description: The buildclt container image to use for the job.
+    build_platform:
+      type: string
+      default: "linux/amd64"
+      description: The target platform to build for. Format "linux/amd64,linux/arm64"
+    push:
+      type: boolean
+      default: true
+      description: If the build image should be pushed.
+    cache:
+      type: boolean
+      default: false
+      description: If caching should be enabled. Currently NOOP
+    image_tags:
+      type: array
+      default:
+        - "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
+      description: List of image tags for the given build.
+    context:
+      type: string
+      default: $CI_PROJECT_DIR
+      description: Path to the build context.
+    dockerfile:
+      type: string
+      default: $CI_PROJECT_DIR/Dockerfile
+      description: Path to the Dockerfile.
+    authors:
+      type: string
+      default: $CI_COMMIT_AUTHOR
+      description: Project Authors.
+    source_url:
+      type: string
+      default: $CI_PROJECT_URL
+      description: Project source code URL.
+    project_url:
+      type: string
+      default: $CI_PROJECT_URL
+      description: Project URL.
+    version:
+      type: string
+      default: $CI_COMMIT_REF_NAME
+      description: Application Version.
+    documentation_url:
+      type: string
+      default: $CI_PROJECT_URL
+      description: Project documentation URL.
+    description:
+      type: string
+      default: $CI_PROJECT_DESCRIPTION
+      description: Project description.
+    vendor:
+      type: string
+      default: UNKNOWN
+      description: Project vendor info.
+    license:
+      type: string
+      default: UNKNOWN
+      description: Project license info.
+    services:
+      type: array
+      default:
+        - name: moby/buildkit
+          entrypoint:
+            - sh
+            - -c
+            - echo "${BUILDKITD_CONF}" > /etc/buildkit/buildkitd.toml && buildkitd
+          alias: buildkitd
+          variables:
+            HEALTHCHECK_TCP_PORT: "1234"
+            BUILDKITD_CONF: |
+              # Buildkitd conf docs: https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md
+              # Listen via tcp
+              [grpc]
+                address = [ "tcp://0.0.0.0:1234" ]
+              # Use the uni-assist nexus registry or google mirror
+              [registry."docker.io"]
+                mirrors = ["mirror.gcr.io"]
+    rules:
+      type: array
+      default:
+        # Add the job to merge request pipelines if there's an open merge request.
+        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+        # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
+        - if: $CI_OPEN_MERGE_REQUESTS
+          when: never
+        # Add the job to main branch pipelines.
+        - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+        - if: $CI_COMMIT_TAG
+---
+$[[ inputs.as_job ]]:
+  stage: $[[ inputs.stage ]]
+  image:
+    name: "$[[ inputs.buildctl_image ]]"
+    entrypoint: [""]
+  services: $[[ inputs.services ]]
+  variables:
+    CONTAINER_BUILD_PUSH: "$[[ inputs.push ]]"
+    CONTAINER_BUILD_USE_CACHE: "$[[ inputs.cache ]]"
+    CONTAINER_BUILD_PLATFORM: "$[[ inputs.build_platform ]]"
+    CONTEXT: "$[[ inputs.context | expand_vars ]]"
+    DOCKERFILE: "$[[ inputs.dockerfile | expand_vars ]]"
+    AUTHORS: "$[[ inputs.authors ]]"
+    SOURCE_URL: "$[[ inputs.source_url ]]"
+    PROJECT_URL: "$[[ inputs.project_url ]]"
+    VERSION: "$[[ inputs.version ]]"
+    DOCUMENTATION_URL: "$[[ inputs.documentation_url ]]"
+    DESCRIPTION: "$[[ inputs.description ]]"
+    VENDOR: "$[[ inputs.vendor ]]"
+    LICENSE: "$[[ inputs.license ]]"
+    BUILDKIT_HOST: tcp://buildkitd:1234
+  before_script:
+    - |
+      echo "Setting up job..." > /dev/null
+      mkdir -p ~/.docker
+    - |
+      echo "Creating login file..." > /dev/null
+      echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > ~/.docker/config.json
+  script:
+    - echo "Using version $(buildctl --version)"
+    - |
+      echo "Setting variables" > /dev/null
+      if [ "${CONTAINER_BUILD_USE_CACHE}" = "true" ]; then export CONTAINER_BUILD_EXTRA_ARGS=${CONTAINER_BUILD_DEFAULT_CACHE_ARGS} ${CONTAINER_BUILD_EXTRA_ARGS}; fi
+
+      # Remove [] from var
+      IMAGE_TAGS="$[[ inputs.image_tags ]]"
+      IMAGE_TAGS="${IMAGE_TAGS#[}"
+      IMAGE_TAGS="${IMAGE_TAGS%]}"
+
+      # Read the cleaned-up string into an array, splitting by commas
+      IFS=","
+      for IMAGE_TAG in ${IMAGE_TAGS}; do
+        # Ensure to strip any whitespace and send a curl request to each URL
+        CONTAINER_BUILD_IMAGE_TAGS="$CONTAINER_BUILD_IMAGE_TAGS --output type=image,name=$IMAGE_TAG,push=$CONTAINER_BUILD_PUSH"
+      done
+      unset IFS
+
+      # Extract the base image name from the Dockerfile
+      if [ -f "$DOCKERFILE" ]; then
+          BASE_IMG_NAME=$(grep "FROM" "$DOCKERFILE" | tail -n 1 | awk '{print $2}')
+          DOCKERFILE_PATH=$(dirname "$DOCKERFILE")
+          DOCKERFILE=$(basename "$DOCKERFILE")
+      else
+          echo "DOCKERFILE $DOCKERFILE does not exist."
+      fi
+
+      echo -e "Building:\n$(echo "$CONTAINER_BUILD_IMAGE_TAGS" | sed 's/image /image\n/g' | tr -d ' ')"
+      echo "Using context \"${CONTEXT}\" with dockerfile \"${DOCKERFILE}\""
+    - |
+      echo "Building image with buildkit..."
+      if [ ! -z ${CONTAINER_BUILD_EXTRA_ARGS+x} ]; then echo "Using extra args=${CONTAINER_BUILD_EXTRA_ARGS}"; fi
+      buildctl build \
+      --frontend dockerfile.v0 \
+      --opt platform="${CONTAINER_BUILD_PLATFORM}" \
+      --opt label:org.opencontainers.image.created="$(date -D FMT=minutes)" \
+      --opt label:org.opencontainers.image.authors="${AUTHORS}" \
+      --opt label:org.opencontainers.image.url="${PROJECT_URL}" \
+      --opt label:org.opencontainers.image.documentation="${DOCUMENTATION_URL}" \
+      --opt label:org.opencontainers.image.source="${SOURCE_URL}" \
+      --opt label:org.opencontainers.image.version="${VERSION}" \
+      --opt label:org.opencontainers.image.revision="${CI_COMMIT_SHORT_SHA}" \
+      --opt label:org.opencontainers.image.vendor="${VENDOR}" \
+      --opt label:org.opencontainers.image.licenses="${LICENSE}" \
+      --opt label:org.opencontainers.image.title="${CI_PROJECT_NAME}" \
+      --opt label:org.opencontainers.image.description="${DESCRIPTION}" \
+      --opt label:org.opencontainers.image.base.name="${BASE_IMG_NAME}" \
+      --opt filename="${DOCKERFILE}" \
+      --local context="${CONTEXT}" \
+      --local dockerfile="${DOCKERFILE_PATH}" \
+      $CONTAINER_BUILD_IMAGE_TAGS $CONTAINER_BUILD_EXTRA_ARGS
+      echo "Build done!"
+  rules: $[[ inputs.rules ]]