From ab134807eae53b303fe8bb054ec5808543fa9624 Mon Sep 17 00:00:00 2001 From: Piotr Banasik Date: Fri, 3 Sep 2021 06:54:10 -0700 Subject: [PATCH] Server: Enable multi platform builds (amd64, armv7 and arm64) (#5338) --- .github/workflows/github-actions-main.yml | 8 ++++ packages/tools/buildServerDocker.ts | 58 +++++++++++++++++++++-- packages/tools/tool-utils.ts | 4 ++ 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/.github/workflows/github-actions-main.yml b/.github/workflows/github-actions-main.yml index e83afc1c1..2e8149d1d 100644 --- a/.github/workflows/github-actions-main.yml +++ b/.github/workflows/github-actions-main.yml @@ -35,6 +35,14 @@ jobs: sudo apt-get update || true sudo apt-get install -y docker-ce docker-ce-cli containerd.io + # the next line enables multi-architecture support for docker, it basically makes it use qemu for non native platforms + # See https://hub.docker.com/r/tonistiigi/binfmt for more info + docker run --privileged --rm tonistiigi/binfmt --install all + + # this just prints the info about what platforms are supported in the builder (can help debugging if something isn't working right) + # and also proves the above worked properly + sudo docker buildx ls + - uses: actions/checkout@v2 - uses: olegtarasov/get-tag@v2.1 - uses: actions/setup-node@v2 diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index 6900548fb..47d7d8250 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -1,6 +1,8 @@ import { execCommand2, rootDir } from './tool-utils'; import * as moment from 'moment'; +const DockerImageName = 'joplin/server'; + function getVersionFromTag(tagName: string, isPreRelease: boolean): string { if (tagName.indexOf('server-') !== 0) throw new Error(`Invalid tag: ${tagName}`); const s = tagName.split('-'); @@ -12,6 +14,10 @@ function getIsPreRelease(tagName: string): boolean { return tagName.indexOf('-beta') > 0; } +function normalizePlatform(platform: string) { + return platform.replace(/\//g, '-'); +} + async function main() { const argv = require('yargs').argv; if (!argv.tagName) throw new Error('--tag-name not provided'); @@ -27,7 +33,11 @@ async function main() { } catch (error) { console.info('Could not get git commit: metadata revision field will be empty'); } - const buildArgs = `--build-arg BUILD_DATE="${buildDate}" --build-arg REVISION="${revision}" --build-arg VERSION="${imageVersion}"`; + const buildArgs = [ + `--build-arg BUILD_DATE="${buildDate}"`, + `--build-arg REVISION="${revision}"`, + `--build-arg VERSION="${imageVersion}"`, + ]; const dockerTags: string[] = []; const versionPart = imageVersion.split('.'); dockerTags.push(isPreRelease ? 'beta' : 'latest'); @@ -44,10 +54,48 @@ async function main() { console.info('isPreRelease:', isPreRelease); console.info('Docker tags:', dockerTags.join(', ')); - await execCommand2(`docker build -t "joplin/server:${imageVersion}" ${buildArgs} -f Dockerfile.server .`); - for (const tag of dockerTags) { - await execCommand2(`docker tag "joplin/server:${imageVersion}" "joplin/server:${tag}"`); - if (pushImages) await execCommand2(`docker push joplin/server:${tag}`); + const platforms = [ + 'linux/amd64', + 'linux/arm64', + 'linux/arm/v7', + ]; + + // this will build a bunch of local image tags named: ${imageVersion}-${platform} with the slashes replaced with dashes + for (const platform of platforms) { + const normalizedPlatform = normalizePlatform(platform); + await execCommand2([ + 'docker', 'build', + '--platform', platform, + '-t', `${DockerImageName}:${imageVersion}-${normalizedPlatform}`, + ...buildArgs, + '-f', 'Dockerfile.server', + '.', + ]); + if (pushImages) { + await execCommand2([ + 'docker', 'push', `${DockerImageName}:${imageVersion}-${normalizedPlatform}`, + ]); + } + } + + // now we have to create the right manifests and push them + if (pushImages) { + for (const tag of dockerTags) { + // manifest create requires the tags being amended in to exist on the remote, so this all can only happen if pushImages is true + const platformArgs: string[] = []; + for (const platform in platforms) { + platformArgs.concat('--amend', `${DockerImageName}:${imageVersion}-${normalizePlatform(platform)}`); + } + await execCommand2([ + 'docker', 'manifest', 'create', + `${DockerImageName}:${tag}`, + ...platformArgs, + ]); + await execCommand2([ + 'docker', 'manifest', 'push', + `${DockerImageName}:${tag}`, + ]); + } } } diff --git a/packages/tools/tool-utils.ts b/packages/tools/tool-utils.ts index 9ef09fd85..48d10de9e 100644 --- a/packages/tools/tool-utils.ts +++ b/packages/tools/tool-utils.ts @@ -147,6 +147,7 @@ export function execCommandVerbose(commandName: string, args: string[] = []) { interface ExecCommandOptions { showInput?: boolean; showOutput?: boolean; + showError?: boolean; quiet?: boolean; } @@ -160,6 +161,7 @@ export async function execCommand2(command: string | string[], options: ExecComm options = { showInput: true, showOutput: true, + showError: true, quiet: false, ...options, }; @@ -167,6 +169,7 @@ export async function execCommand2(command: string | string[], options: ExecComm if (options.quiet) { options.showInput = false; options.showOutput = false; + options.showError = false; } if (options.showInput) { @@ -182,6 +185,7 @@ export async function execCommand2(command: string | string[], options: ExecComm args.splice(0, 1); const promise = execa(executableName, args); if (options.showOutput) promise.stdout.pipe(process.stdout); + if (options.showError) promise.stderr.pipe(process.stderr); const result = await promise; return result.stdout.trim(); }