Fix multi-architecture unit testing.

The Github action we were using for multi-architecture testing stopped working. The project does not seem to be getting regular maintenance so it seems better to roll multi-architecture testing into our existing container builds. Introduce multi-architecture builds and testing into our test framework. For now this only works for unit tests -- integration tests will still only run on x86_64. That could be updated in the future but since emulation is so slow it is not clear if it would be useful. Also fix an invalid 32-bit checksum. The d11 test had not been running as 32-bit since d8ff89a so the checksum was not updated when it should have been in 48f511d.
2025-10-30 23:37:45 +02:00 · 2025-02-28 15:38:05 -06:00
parent 814bf487c3
commit 57ffd2dfcb
8 changed files with 83 additions and 50 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -75,13 +75,13 @@ jobs:
  # expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to
  # test on one big-endian architecture to be sure that checksums are correct.
  arch:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04

    strategy:
      matrix:
        include:
-          - image: '--platform=linux/ppc64le ubuntu:22.04'
-          - image: '--platform=linux/s390x ubuntu:22.04'
+          - arch: 'ppc64le'
+          - arch: 's390x'

    steps:
      - name: Checkout Code
@@ -89,23 +89,19 @@ jobs:
        with:
          path: pgbackrest

+      - name: Install
+        run: |
+          sudo apt-get update
+          sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
+          docker run --privileged --rm tonistiigi/binfmt --install all
+
+      - name: Build VM
+        run: ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm-build --vm=u22 --vm-arch=${{matrix.arch}}
+
      - name: Run Test
-        uses: uraimo/run-on-arch-action@v2
-        id: runcmd
-        with:
-          base_image: ${{matrix.image}}
-
-          # Cache builds
-          githubToken: ${{github.token}}
-
-          install: |
-            apt-get update
-            DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache python3-distutils meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
-
-          run: |
-            git config --global --add safe.directory ${GITHUB_WORKSPACE?}/pgbackrest
-            ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
-            ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface
+        run: |
+          ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
+          ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface

  # Run meson unity build to check for errors, unused functions, and externed functions
  unity:
--- a/test/container.yaml
+++ b/test/container.yaml
@@ -10,11 +10,17 @@
 #
 # To upload a new image:
 # - docker login -u pgbackrest
-# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
+# - DATE=YYYYMMDDX;VM=X;ARCH=X;BASE=pgbackrest/test:${VM?}-base-${ARCH?};docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
 # **********************************************************************************************************************************
-20250123A:
+20250228A:
+  ppc64le:
+    u22: 28fa02cb370bbdacadf984afb215d3973ed5ab3e
+
+  s390x:
+    u22: f3e108dd7f808f6b0e82eadcfd72f585a7632530
+
  x86_64:
-    d11: f3bc523f10e873f85b889120ea08c6c53358cc47
+    d11: 01e6970744c2b2529a14832e92cb861c7da94308
    f41: ce870455184e991e0efd90176da1412f0f3f72a2
    rh8: 4d141c845abfbdbf402ba447cf2bd2e4357c8a63
    u20: 862159b4d2169a4752b106639ca0f47c1ebb1f86
--- a/test/lib/pgBackRestTest/Common/ContainerTest.pm
+++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm
@@ -84,6 +84,7 @@ sub containerWrite
    my $oStorageDocker = shift;
    my $strTempPath = shift;
    my $strOS = shift;
+    my $strArch = shift;
    my $strTitle = shift;
    my $strImageParent = shift;
    my $strImage = shift;
@@ -104,14 +105,16 @@ sub containerWrite
    my $strScriptSha1;
    my $bCached = false;

-    if ($strImage =~ /\-base$/)
+    if ($strImage =~ /\-base\-/)
    {
        $strScriptSha1 = sha1_hex($strScript);

        foreach my $strBuild (reverse(keys(%{$hContainerCache})))
        {
-            if (defined($hContainerCache->{$strBuild}{hostArch()}{$strOS}) &&
-                $hContainerCache->{$strBuild}{hostArch()}{$strOS} eq $strScriptSha1)
+            my $strArchLookup = defined($strArch) ? $strArch : hostArch();
+
+            if (defined($hContainerCache->{$strBuild}{$strArchLookup}{$strOS}) &&
+                $hContainerCache->{$strBuild}{$strArchLookup}{$strOS} eq $strScriptSha1)
            {
                &log(INFO, "Using cached ${strTag}-${strBuild} image (${strScriptSha1}) ...");

@@ -132,7 +135,8 @@ sub containerWrite
    # Write the image
    $oStorageDocker->put("${strTempPath}/${strImage}", trim($strScript) . "\n");
    executeTest(
-        'docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
+        'docker build' . (defined($strArch) ? " --platform linux/${strArch}" : '') .
+        (defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
            $oStorageDocker->pathGet('test'),
        {bSuppressStdErr => true, bShowOutputAsync => (logLevel())[1] eq DETAIL});
 }
@@ -338,6 +342,7 @@ sub containerBuild
 {
    my $oStorageDocker = shift;
    my $strVm = shift;
+    my $strArch = shift;
    my $bVmForce = shift;

    # Create temp path
@@ -378,8 +383,10 @@ sub containerBuild

        # Base image
        ###########################################################################################################################
-        my $strImageParent = "$$oVm{$strOS}{&VM_IMAGE}";
-        my $strImage = "${strOS}-base";
+        my $strImageParent =
+            (defined($strArch) ? "${strArch}/" : (vmArch($strOS) eq VM_ARCH_AMD64 ? '' : vmArch($strOS) . '/')) .
+            "$$oVm{$strOS}{&VM_IMAGE}";
+        my $strImage = "${strOS}-base" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());
        my $strCopy = undef;

        #---------------------------------------------------------------------------------------------------------------------------
@@ -559,14 +566,14 @@ sub containerBuild
        }

        containerWrite(
-            $oStorageDocker, $strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
+            $oStorageDocker, $strTempPath, $strOS, $strArch, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);

        # Test image
        ########################################################################################################################
        if (!$bDeprecated)
        {
-            $strImageParent = containerRepo() . ":${strOS}-base";
-            $strImage = "${strOS}-test";
+            $strImageParent = containerRepo() . ":${strImage}";
+            $strImage = "${strOS}-test" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());

            $strCopy = undef;
            $strScript = '';
@@ -641,7 +648,8 @@ sub containerBuild
            $strScript .= entryPointSetup($strOS);

            containerWrite(
-                $oStorageDocker, $strTempPath, $strOS, 'Test', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
+                $oStorageDocker, $strTempPath, $strOS, $strArch, 'Test', $strImageParent, $strImage, $strCopy, $strScript,
+                $bVmForce);
        }
    }

--- a/test/lib/pgBackRestTest/Common/JobTest.pm
+++ b/test/lib/pgBackRestTest/Common/JobTest.pm
@@ -51,6 +51,8 @@ sub new
        $self->{oTest},
        $self->{bDryRun},
        $self->{bVmOut},
+        $self->{strPlatform},
+        $self->{strImage},
        $self->{iVmIdx},
        $self->{iVmMax},
        $self->{strMakeCmd},
@@ -84,6 +86,8 @@ sub new
            {name => 'oTest'},
            {name => 'bDryRun'},
            {name => 'bVmOut'},
+            {name => 'strPlatform'},
+            {name => 'strImage'},
            {name => 'iVmIdx'},
            {name => 'iVmMax'},
            {name => 'strMakeCmd'},
@@ -207,15 +211,15 @@ sub run
                    my $strBuildPath = $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM};

                    executeTest(
-                        'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" .
-                        " -v ${strHostTestPath}:${strVmTestPath}" .
+                        'docker run' . $self->{strPlatform} . ' -itd -h ' .
+                        $self->{oTest}->{&TEST_VM} . "-test --name=${strImage} -v ${strHostTestPath}:${strVmTestPath}" .
                        ($self->{oTest}->{&TEST_C} ? " -v $self->{strUnitPath}:$self->{strUnitPath}" : '') .
                        ($self->{oTest}->{&TEST_C} ? " -v $self->{strDataPath}:$self->{strDataPath}" : '') .
                        " -v $self->{strBackRestBase}:$self->{strBackRestBase}" .
                        " -v $self->{strRepoPath}:$self->{strRepoPath}" .
                        ($self->{oTest}->{&TEST_C} ? " -v ${strBuildPath}:${strBuildPath}:ro" : '') .
                        ($self->{oTest}->{&TEST_C} ? " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" : '') .
-                        ' ' . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . '-test',
+                        ' ' . $self->{strImage},
                        {bSuppressStdErr => true});
                }
            }
--- a/test/lib/pgBackRestTest/Common/VmTest.pm
+++ b/test/lib/pgBackRestTest/Common/VmTest.pm
@@ -379,6 +379,18 @@ sub vmCoverageC

 push @EXPORT, qw(vmCoverageC);

+####################################################################################################################################
+# Get vm architecture
+####################################################################################################################################
+sub vmArch
+{
+    my $strVm = shift;
+
+    return $oyVm->{$strVm}{&VM_ARCH};
+}
+
+push @EXPORT, qw(vmArch);
+
 ####################################################################################################################################
 # Get vm architecture bits
 ####################################################################################################################################
@@ -386,7 +398,7 @@ sub vmArchBits
 {
    my $strVm = shift;

-    return ($oyVm->{$strVm}{&VM_ARCH} eq VM_ARCH_I386 ? 32 : 64);
+    return (vmArch($strVm) eq VM_ARCH_I386 ? 32 : 64);
 }

 push @EXPORT, qw(vmArchBits);
--- a/test/src/common/harnessHost.c
+++ b/test/src/common/harnessHost.c
@@ -1128,7 +1128,7 @@ hrnHostBuildRun(const int line, const StringId id)
        const bool isPg = strBeginsWithZ(name, "pg");
        const bool isRepo = id == hrnHostLocal.repoHost;
        const String *const container = strNewFmt("test-%u-%s", testIdx(), strZ(name));
-        const String *const image = strNewFmt("pgbackrest/test:%s-test", testVm());
+        const String *const image = strNewFmt("pgbackrest/test:%s-test-x86_64", testVm());
        const String *const dataPath = strNewFmt("%s/%s", testPath(), strZ(name));
        String *const option = strNewFmt(
            "-v '%s/cfg:/etc/pgbackrest:ro' -v '%s:/usr/bin/pgbackrest:ro' -v '%s:%s:ro'", strZ(dataPath), testProjectExe(),
@@ -1330,7 +1330,8 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si
                    MEM_CONTEXT_PRIOR_BEGIN()
                    {
                        hrnHostNewP(
-                            HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test", testVm()), .noUpdateHosts = true);
+                            HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test-x86_64", testVm()),
+                            .noUpdateHosts = true);
                    }
                    MEM_CONTEXT_PRIOR_END();

--- a/test/src/module/command/backupTest.c
+++ b/test/src/module/command/backupTest.c
@@ -1951,7 +1951,7 @@ testRun(void)
            "P00   INFO: full backup size = 8KB, file total = 2",
            TEST_64BIT() ?
                (TEST_BIG_ENDIAN() ? "ead3f998dc6dbc4b444f89cd449dcb81801a21ed" : "6f7fb3cd71dbef602850d05332cdd1c8e4a64121") :
-                "56d0a8a2d6b0b6dd2880c4b0221fec24e958c1a6");
+                "a90290adaf2e8a36d53b0c15d277e23e1b321fdb");

        // Make pg no longer appear to be running
        HRN_STORAGE_REMOVE(storagePgWrite(), PG_FILE_POSTMTRPID, .errorOnMissing = true);
--- a/test/test.pl
+++ b/test/test.pl
@@ -107,6 +107,7 @@ test.pl [options]

 VM Options:
   --vm                 docker container to build/test (e.g. rh8)
+   --vm-arch            docker container architecture
   --vm-build           build Docker containers
   --vm-force           force a rebuild of Docker containers
   --vm-out             Show VM output (default false)
@@ -143,6 +144,7 @@ my $bHelp = false;
 my $bQuiet = false;
 my $strPgVersion = 'minimal';
 my $strVm = VM_NONE;
+my $strVmArch;
 my $bVmBuild = false;
 my $bVmForce = false;
 my $bBuildOnly = false;
@@ -183,6 +185,7 @@ GetOptions ('q|quiet' => \$bQuiet,
            'log-level-test-file=s' => \$strLogLevelTestFile,
            'no-log-timestamp' => \$bNoLogTimestamp,
            'vm=s' => \$strVm,
+            'vm-arch=s' => \$strVmArch,
            'vm-out' => \$bVmOut,
            'vm-build' => \$bVmBuild,
            'vm-force' => \$bVmForce,
@@ -364,7 +367,7 @@ eval
    ################################################################################################################################
    if ($bVmBuild)
    {
-        containerBuild($oStorageBackRest, $strVm, $bVmForce);
+        containerBuild($oStorageBackRest, $strVm, $strVmArch, $bVmForce);
        exit 0;
    }

@@ -454,6 +457,9 @@ eval

    # Start build container if vm is not none
    #-------------------------------------------------------------------------------------------------------------------------------
+    my $strPlatform = defined($strVmArch) ? " --platform linux/${strVmArch}" : '';
+    my $strImage = containerRepo() . ":${strVm}-test" . (defined($strVmArch) ? "-${strVmArch}" : '-' . hostArch());
+
    if ($strVm ne VM_NONE)
    {
        my $strCCachePath = "${strTestPath}/ccache-0/${strVm}";
@@ -464,9 +470,9 @@ eval
        }

        executeTest(
-            "docker run -itd -h test-build --name=test-build" .
+            "docker run${strPlatform} -itd -h test-build --name=test-build" .
                " -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath}" .
-                " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . ' ' . containerRepo() . ":${strVm}-test",
+                " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . " ${strImage}",
            {bSuppressStdErr => true});
    }

@@ -694,8 +700,8 @@ eval
                    if ($strVm ne VM_NONE)
                    {
                        executeTest(
-                            "docker run -itd -h test-build --name=test-build" .
-                            " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test",
+                            "docker run${strPlatform} -itd -h test-build --name=test-build" .
+                            " -v ${strBackRestBase}:${strBackRestBase} ${strImage}",
                            {bSuppressStdErr => true});
                    }

@@ -778,8 +784,8 @@ eval
                    if ($strVm ne VM_NONE)
                    {
                        executeTest(
-                            "docker run -itd -h test-build --name=test-build" .
-                            " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test",
+                            "docker run${strPlatform} -itd -h test-build --name=test-build" .
+                            " -v ${strBackRestBase}:${strBackRestBase} ${strImage}",
                            {bSuppressStdErr => true});
                    }

@@ -958,10 +964,10 @@ eval
            if (!defined($$oyProcess[$iVmIdx]) && $iTestIdx < @{$oyTestRun})
            {
                my $oJob = new pgBackRestTest::Common::JobTest(
-                    $oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, $iVmMax,
-                    $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, $strLogLevelTestFile, !$bNoLogTimestamp,
-                    $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind, !$bNoCoverage, $bCoverageSummary,
-                    !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace,
+                    $oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $strPlatform,
+                    $strImage, $iVmIdx, $iVmMax, $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest,
+                    $strLogLevelTestFile, !$bNoLogTimestamp, $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind,
+                    !$bNoCoverage, $bCoverageSummary, !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace,
                    $iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax));
                $iTestIdx++;