1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-15 01:04:37 +02:00

Fix multi-architecture unit testing.

The Github action we were using for multi-architecture testing stopped working. The project does not seem to be getting regular maintenance so it seems better to roll multi-architecture testing into our existing container builds.

Introduce multi-architecture builds and testing into our test framework. For now this only works for unit tests -- integration tests will still only run on x86_64. That could be updated in the future but since emulation is so slow it is not clear if it would be useful.

Also fix an invalid 32-bit checksum. The d11 test had not been running as 32-bit since d8ff89a so the checksum was not updated when it should have been in 48f511d.
This commit is contained in:
David Steele
2025-02-28 15:38:05 -06:00
parent 814bf487c3
commit 57ffd2dfcb
8 changed files with 83 additions and 50 deletions

View File

@ -75,13 +75,13 @@ jobs:
# expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to
# test on one big-endian architecture to be sure that checksums are correct.
arch:
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
strategy:
matrix:
include:
- image: '--platform=linux/ppc64le ubuntu:22.04'
- image: '--platform=linux/s390x ubuntu:22.04'
- arch: 'ppc64le'
- arch: 's390x'
steps:
- name: Checkout Code
@ -89,23 +89,19 @@ jobs:
with:
path: pgbackrest
- name: Install
run: |
sudo apt-get update
sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
docker run --privileged --rm tonistiigi/binfmt --install all
- name: Build VM
run: ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm-build --vm=u22 --vm-arch=${{matrix.arch}}
- name: Run Test
uses: uraimo/run-on-arch-action@v2
id: runcmd
with:
base_image: ${{matrix.image}}
# Cache builds
githubToken: ${{github.token}}
install: |
apt-get update
DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache python3-distutils meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE?}/pgbackrest
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface
run: |
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface
# Run meson unity build to check for errors, unused functions, and externed functions
unity:

View File

@ -10,11 +10,17 @@
#
# To upload a new image:
# - docker login -u pgbackrest
# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
# - DATE=YYYYMMDDX;VM=X;ARCH=X;BASE=pgbackrest/test:${VM?}-base-${ARCH?};docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
# **********************************************************************************************************************************
20250123A:
20250228A:
ppc64le:
u22: 28fa02cb370bbdacadf984afb215d3973ed5ab3e
s390x:
u22: f3e108dd7f808f6b0e82eadcfd72f585a7632530
x86_64:
d11: f3bc523f10e873f85b889120ea08c6c53358cc47
d11: 01e6970744c2b2529a14832e92cb861c7da94308
f41: ce870455184e991e0efd90176da1412f0f3f72a2
rh8: 4d141c845abfbdbf402ba447cf2bd2e4357c8a63
u20: 862159b4d2169a4752b106639ca0f47c1ebb1f86

View File

@ -84,6 +84,7 @@ sub containerWrite
my $oStorageDocker = shift;
my $strTempPath = shift;
my $strOS = shift;
my $strArch = shift;
my $strTitle = shift;
my $strImageParent = shift;
my $strImage = shift;
@ -104,14 +105,16 @@ sub containerWrite
my $strScriptSha1;
my $bCached = false;
if ($strImage =~ /\-base$/)
if ($strImage =~ /\-base\-/)
{
$strScriptSha1 = sha1_hex($strScript);
foreach my $strBuild (reverse(keys(%{$hContainerCache})))
{
if (defined($hContainerCache->{$strBuild}{hostArch()}{$strOS}) &&
$hContainerCache->{$strBuild}{hostArch()}{$strOS} eq $strScriptSha1)
my $strArchLookup = defined($strArch) ? $strArch : hostArch();
if (defined($hContainerCache->{$strBuild}{$strArchLookup}{$strOS}) &&
$hContainerCache->{$strBuild}{$strArchLookup}{$strOS} eq $strScriptSha1)
{
&log(INFO, "Using cached ${strTag}-${strBuild} image (${strScriptSha1}) ...");
@ -132,7 +135,8 @@ sub containerWrite
# Write the image
$oStorageDocker->put("${strTempPath}/${strImage}", trim($strScript) . "\n");
executeTest(
'docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
'docker build' . (defined($strArch) ? " --platform linux/${strArch}" : '') .
(defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
$oStorageDocker->pathGet('test'),
{bSuppressStdErr => true, bShowOutputAsync => (logLevel())[1] eq DETAIL});
}
@ -338,6 +342,7 @@ sub containerBuild
{
my $oStorageDocker = shift;
my $strVm = shift;
my $strArch = shift;
my $bVmForce = shift;
# Create temp path
@ -378,8 +383,10 @@ sub containerBuild
# Base image
###########################################################################################################################
my $strImageParent = "$$oVm{$strOS}{&VM_IMAGE}";
my $strImage = "${strOS}-base";
my $strImageParent =
(defined($strArch) ? "${strArch}/" : (vmArch($strOS) eq VM_ARCH_AMD64 ? '' : vmArch($strOS) . '/')) .
"$$oVm{$strOS}{&VM_IMAGE}";
my $strImage = "${strOS}-base" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());
my $strCopy = undef;
#---------------------------------------------------------------------------------------------------------------------------
@ -559,14 +566,14 @@ sub containerBuild
}
containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
$oStorageDocker, $strTempPath, $strOS, $strArch, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
# Test image
########################################################################################################################
if (!$bDeprecated)
{
$strImageParent = containerRepo() . ":${strOS}-base";
$strImage = "${strOS}-test";
$strImageParent = containerRepo() . ":${strImage}";
$strImage = "${strOS}-test" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());
$strCopy = undef;
$strScript = '';
@ -641,7 +648,8 @@ sub containerBuild
$strScript .= entryPointSetup($strOS);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Test', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
$oStorageDocker, $strTempPath, $strOS, $strArch, 'Test', $strImageParent, $strImage, $strCopy, $strScript,
$bVmForce);
}
}

View File

@ -51,6 +51,8 @@ sub new
$self->{oTest},
$self->{bDryRun},
$self->{bVmOut},
$self->{strPlatform},
$self->{strImage},
$self->{iVmIdx},
$self->{iVmMax},
$self->{strMakeCmd},
@ -84,6 +86,8 @@ sub new
{name => 'oTest'},
{name => 'bDryRun'},
{name => 'bVmOut'},
{name => 'strPlatform'},
{name => 'strImage'},
{name => 'iVmIdx'},
{name => 'iVmMax'},
{name => 'strMakeCmd'},
@ -207,15 +211,15 @@ sub run
my $strBuildPath = $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM};
executeTest(
'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" .
" -v ${strHostTestPath}:${strVmTestPath}" .
'docker run' . $self->{strPlatform} . ' -itd -h ' .
$self->{oTest}->{&TEST_VM} . "-test --name=${strImage} -v ${strHostTestPath}:${strVmTestPath}" .
($self->{oTest}->{&TEST_C} ? " -v $self->{strUnitPath}:$self->{strUnitPath}" : '') .
($self->{oTest}->{&TEST_C} ? " -v $self->{strDataPath}:$self->{strDataPath}" : '') .
" -v $self->{strBackRestBase}:$self->{strBackRestBase}" .
" -v $self->{strRepoPath}:$self->{strRepoPath}" .
($self->{oTest}->{&TEST_C} ? " -v ${strBuildPath}:${strBuildPath}:ro" : '') .
($self->{oTest}->{&TEST_C} ? " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" : '') .
' ' . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . '-test',
' ' . $self->{strImage},
{bSuppressStdErr => true});
}
}

View File

@ -379,6 +379,18 @@ sub vmCoverageC
push @EXPORT, qw(vmCoverageC);
####################################################################################################################################
# Get vm architecture
####################################################################################################################################
sub vmArch
{
my $strVm = shift;
return $oyVm->{$strVm}{&VM_ARCH};
}
push @EXPORT, qw(vmArch);
####################################################################################################################################
# Get vm architecture bits
####################################################################################################################################
@ -386,7 +398,7 @@ sub vmArchBits
{
my $strVm = shift;
return ($oyVm->{$strVm}{&VM_ARCH} eq VM_ARCH_I386 ? 32 : 64);
return (vmArch($strVm) eq VM_ARCH_I386 ? 32 : 64);
}
push @EXPORT, qw(vmArchBits);

View File

@ -1128,7 +1128,7 @@ hrnHostBuildRun(const int line, const StringId id)
const bool isPg = strBeginsWithZ(name, "pg");
const bool isRepo = id == hrnHostLocal.repoHost;
const String *const container = strNewFmt("test-%u-%s", testIdx(), strZ(name));
const String *const image = strNewFmt("pgbackrest/test:%s-test", testVm());
const String *const image = strNewFmt("pgbackrest/test:%s-test-x86_64", testVm());
const String *const dataPath = strNewFmt("%s/%s", testPath(), strZ(name));
String *const option = strNewFmt(
"-v '%s/cfg:/etc/pgbackrest:ro' -v '%s:/usr/bin/pgbackrest:ro' -v '%s:%s:ro'", strZ(dataPath), testProjectExe(),
@ -1330,7 +1330,8 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si
MEM_CONTEXT_PRIOR_BEGIN()
{
hrnHostNewP(
HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test", testVm()), .noUpdateHosts = true);
HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test-x86_64", testVm()),
.noUpdateHosts = true);
}
MEM_CONTEXT_PRIOR_END();

View File

@ -1951,7 +1951,7 @@ testRun(void)
"P00 INFO: full backup size = 8KB, file total = 2",
TEST_64BIT() ?
(TEST_BIG_ENDIAN() ? "ead3f998dc6dbc4b444f89cd449dcb81801a21ed" : "6f7fb3cd71dbef602850d05332cdd1c8e4a64121") :
"56d0a8a2d6b0b6dd2880c4b0221fec24e958c1a6");
"a90290adaf2e8a36d53b0c15d277e23e1b321fdb");
// Make pg no longer appear to be running
HRN_STORAGE_REMOVE(storagePgWrite(), PG_FILE_POSTMTRPID, .errorOnMissing = true);

View File

@ -107,6 +107,7 @@ test.pl [options]
VM Options:
--vm docker container to build/test (e.g. rh8)
--vm-arch docker container architecture
--vm-build build Docker containers
--vm-force force a rebuild of Docker containers
--vm-out Show VM output (default false)
@ -143,6 +144,7 @@ my $bHelp = false;
my $bQuiet = false;
my $strPgVersion = 'minimal';
my $strVm = VM_NONE;
my $strVmArch;
my $bVmBuild = false;
my $bVmForce = false;
my $bBuildOnly = false;
@ -183,6 +185,7 @@ GetOptions ('q|quiet' => \$bQuiet,
'log-level-test-file=s' => \$strLogLevelTestFile,
'no-log-timestamp' => \$bNoLogTimestamp,
'vm=s' => \$strVm,
'vm-arch=s' => \$strVmArch,
'vm-out' => \$bVmOut,
'vm-build' => \$bVmBuild,
'vm-force' => \$bVmForce,
@ -364,7 +367,7 @@ eval
################################################################################################################################
if ($bVmBuild)
{
containerBuild($oStorageBackRest, $strVm, $bVmForce);
containerBuild($oStorageBackRest, $strVm, $strVmArch, $bVmForce);
exit 0;
}
@ -454,6 +457,9 @@ eval
# Start build container if vm is not none
#-------------------------------------------------------------------------------------------------------------------------------
my $strPlatform = defined($strVmArch) ? " --platform linux/${strVmArch}" : '';
my $strImage = containerRepo() . ":${strVm}-test" . (defined($strVmArch) ? "-${strVmArch}" : '-' . hostArch());
if ($strVm ne VM_NONE)
{
my $strCCachePath = "${strTestPath}/ccache-0/${strVm}";
@ -464,9 +470,9 @@ eval
}
executeTest(
"docker run -itd -h test-build --name=test-build" .
"docker run${strPlatform} -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath}" .
" -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . ' ' . containerRepo() . ":${strVm}-test",
" -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . " ${strImage}",
{bSuppressStdErr => true});
}
@ -694,8 +700,8 @@ eval
if ($strVm ne VM_NONE)
{
executeTest(
"docker run -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test",
"docker run${strPlatform} -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} ${strImage}",
{bSuppressStdErr => true});
}
@ -778,8 +784,8 @@ eval
if ($strVm ne VM_NONE)
{
executeTest(
"docker run -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test",
"docker run${strPlatform} -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} ${strImage}",
{bSuppressStdErr => true});
}
@ -958,10 +964,10 @@ eval
if (!defined($$oyProcess[$iVmIdx]) && $iTestIdx < @{$oyTestRun})
{
my $oJob = new pgBackRestTest::Common::JobTest(
$oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, $iVmMax,
$strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, $strLogLevelTestFile, !$bNoLogTimestamp,
$bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind, !$bNoCoverage, $bCoverageSummary,
!$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace,
$oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $strPlatform,
$strImage, $iVmIdx, $iVmMax, $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest,
$strLogLevelTestFile, !$bNoLogTimestamp, $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind,
!$bNoCoverage, $bCoverageSummary, !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace,
$iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax));
$iTestIdx++;