You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-15 01:04:37 +02:00
Fix multi-architecture unit testing.
The Github action we were using for multi-architecture testing stopped working. The project does not seem to be getting regular maintenance so it seems better to roll multi-architecture testing into our existing container builds. Introduce multi-architecture builds and testing into our test framework. For now this only works for unit tests -- integration tests will still only run on x86_64. That could be updated in the future but since emulation is so slow it is not clear if it would be useful. Also fix an invalid 32-bit checksum. The d11 test had not been running as 32-bit sinced8ff89a
so the checksum was not updated when it should have been in48f511d
.
This commit is contained in:
34
.github/workflows/test.yml
vendored
34
.github/workflows/test.yml
vendored
@ -75,13 +75,13 @@ jobs:
|
||||
# expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to
|
||||
# test on one big-endian architecture to be sure that checksums are correct.
|
||||
arch:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- image: '--platform=linux/ppc64le ubuntu:22.04'
|
||||
- image: '--platform=linux/s390x ubuntu:22.04'
|
||||
- arch: 'ppc64le'
|
||||
- arch: 's390x'
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@ -89,23 +89,19 @@ jobs:
|
||||
with:
|
||||
path: pgbackrest
|
||||
|
||||
- name: Install
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
|
||||
docker run --privileged --rm tonistiigi/binfmt --install all
|
||||
|
||||
- name: Build VM
|
||||
run: ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm-build --vm=u22 --vm-arch=${{matrix.arch}}
|
||||
|
||||
- name: Run Test
|
||||
uses: uraimo/run-on-arch-action@v2
|
||||
id: runcmd
|
||||
with:
|
||||
base_image: ${{matrix.image}}
|
||||
|
||||
# Cache builds
|
||||
githubToken: ${{github.token}}
|
||||
|
||||
install: |
|
||||
apt-get update
|
||||
DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache python3-distutils meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
|
||||
|
||||
run: |
|
||||
git config --global --add safe.directory ${GITHUB_WORKSPACE?}/pgbackrest
|
||||
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
|
||||
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface
|
||||
run: |
|
||||
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
|
||||
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface
|
||||
|
||||
# Run meson unity build to check for errors, unused functions, and externed functions
|
||||
unity:
|
||||
|
@ -10,11 +10,17 @@
|
||||
#
|
||||
# To upload a new image:
|
||||
# - docker login -u pgbackrest
|
||||
# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
|
||||
# - DATE=YYYYMMDDX;VM=X;ARCH=X;BASE=pgbackrest/test:${VM?}-base-${ARCH?};docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
|
||||
# **********************************************************************************************************************************
|
||||
20250123A:
|
||||
20250228A:
|
||||
ppc64le:
|
||||
u22: 28fa02cb370bbdacadf984afb215d3973ed5ab3e
|
||||
|
||||
s390x:
|
||||
u22: f3e108dd7f808f6b0e82eadcfd72f585a7632530
|
||||
|
||||
x86_64:
|
||||
d11: f3bc523f10e873f85b889120ea08c6c53358cc47
|
||||
d11: 01e6970744c2b2529a14832e92cb861c7da94308
|
||||
f41: ce870455184e991e0efd90176da1412f0f3f72a2
|
||||
rh8: 4d141c845abfbdbf402ba447cf2bd2e4357c8a63
|
||||
u20: 862159b4d2169a4752b106639ca0f47c1ebb1f86
|
||||
|
@ -84,6 +84,7 @@ sub containerWrite
|
||||
my $oStorageDocker = shift;
|
||||
my $strTempPath = shift;
|
||||
my $strOS = shift;
|
||||
my $strArch = shift;
|
||||
my $strTitle = shift;
|
||||
my $strImageParent = shift;
|
||||
my $strImage = shift;
|
||||
@ -104,14 +105,16 @@ sub containerWrite
|
||||
my $strScriptSha1;
|
||||
my $bCached = false;
|
||||
|
||||
if ($strImage =~ /\-base$/)
|
||||
if ($strImage =~ /\-base\-/)
|
||||
{
|
||||
$strScriptSha1 = sha1_hex($strScript);
|
||||
|
||||
foreach my $strBuild (reverse(keys(%{$hContainerCache})))
|
||||
{
|
||||
if (defined($hContainerCache->{$strBuild}{hostArch()}{$strOS}) &&
|
||||
$hContainerCache->{$strBuild}{hostArch()}{$strOS} eq $strScriptSha1)
|
||||
my $strArchLookup = defined($strArch) ? $strArch : hostArch();
|
||||
|
||||
if (defined($hContainerCache->{$strBuild}{$strArchLookup}{$strOS}) &&
|
||||
$hContainerCache->{$strBuild}{$strArchLookup}{$strOS} eq $strScriptSha1)
|
||||
{
|
||||
&log(INFO, "Using cached ${strTag}-${strBuild} image (${strScriptSha1}) ...");
|
||||
|
||||
@ -132,7 +135,8 @@ sub containerWrite
|
||||
# Write the image
|
||||
$oStorageDocker->put("${strTempPath}/${strImage}", trim($strScript) . "\n");
|
||||
executeTest(
|
||||
'docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
|
||||
'docker build' . (defined($strArch) ? " --platform linux/${strArch}" : '') .
|
||||
(defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
|
||||
$oStorageDocker->pathGet('test'),
|
||||
{bSuppressStdErr => true, bShowOutputAsync => (logLevel())[1] eq DETAIL});
|
||||
}
|
||||
@ -338,6 +342,7 @@ sub containerBuild
|
||||
{
|
||||
my $oStorageDocker = shift;
|
||||
my $strVm = shift;
|
||||
my $strArch = shift;
|
||||
my $bVmForce = shift;
|
||||
|
||||
# Create temp path
|
||||
@ -378,8 +383,10 @@ sub containerBuild
|
||||
|
||||
# Base image
|
||||
###########################################################################################################################
|
||||
my $strImageParent = "$$oVm{$strOS}{&VM_IMAGE}";
|
||||
my $strImage = "${strOS}-base";
|
||||
my $strImageParent =
|
||||
(defined($strArch) ? "${strArch}/" : (vmArch($strOS) eq VM_ARCH_AMD64 ? '' : vmArch($strOS) . '/')) .
|
||||
"$$oVm{$strOS}{&VM_IMAGE}";
|
||||
my $strImage = "${strOS}-base" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());
|
||||
my $strCopy = undef;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@ -559,14 +566,14 @@ sub containerBuild
|
||||
}
|
||||
|
||||
containerWrite(
|
||||
$oStorageDocker, $strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
|
||||
$oStorageDocker, $strTempPath, $strOS, $strArch, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
|
||||
|
||||
# Test image
|
||||
########################################################################################################################
|
||||
if (!$bDeprecated)
|
||||
{
|
||||
$strImageParent = containerRepo() . ":${strOS}-base";
|
||||
$strImage = "${strOS}-test";
|
||||
$strImageParent = containerRepo() . ":${strImage}";
|
||||
$strImage = "${strOS}-test" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());
|
||||
|
||||
$strCopy = undef;
|
||||
$strScript = '';
|
||||
@ -641,7 +648,8 @@ sub containerBuild
|
||||
$strScript .= entryPointSetup($strOS);
|
||||
|
||||
containerWrite(
|
||||
$oStorageDocker, $strTempPath, $strOS, 'Test', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
|
||||
$oStorageDocker, $strTempPath, $strOS, $strArch, 'Test', $strImageParent, $strImage, $strCopy, $strScript,
|
||||
$bVmForce);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -51,6 +51,8 @@ sub new
|
||||
$self->{oTest},
|
||||
$self->{bDryRun},
|
||||
$self->{bVmOut},
|
||||
$self->{strPlatform},
|
||||
$self->{strImage},
|
||||
$self->{iVmIdx},
|
||||
$self->{iVmMax},
|
||||
$self->{strMakeCmd},
|
||||
@ -84,6 +86,8 @@ sub new
|
||||
{name => 'oTest'},
|
||||
{name => 'bDryRun'},
|
||||
{name => 'bVmOut'},
|
||||
{name => 'strPlatform'},
|
||||
{name => 'strImage'},
|
||||
{name => 'iVmIdx'},
|
||||
{name => 'iVmMax'},
|
||||
{name => 'strMakeCmd'},
|
||||
@ -207,15 +211,15 @@ sub run
|
||||
my $strBuildPath = $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM};
|
||||
|
||||
executeTest(
|
||||
'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" .
|
||||
" -v ${strHostTestPath}:${strVmTestPath}" .
|
||||
'docker run' . $self->{strPlatform} . ' -itd -h ' .
|
||||
$self->{oTest}->{&TEST_VM} . "-test --name=${strImage} -v ${strHostTestPath}:${strVmTestPath}" .
|
||||
($self->{oTest}->{&TEST_C} ? " -v $self->{strUnitPath}:$self->{strUnitPath}" : '') .
|
||||
($self->{oTest}->{&TEST_C} ? " -v $self->{strDataPath}:$self->{strDataPath}" : '') .
|
||||
" -v $self->{strBackRestBase}:$self->{strBackRestBase}" .
|
||||
" -v $self->{strRepoPath}:$self->{strRepoPath}" .
|
||||
($self->{oTest}->{&TEST_C} ? " -v ${strBuildPath}:${strBuildPath}:ro" : '') .
|
||||
($self->{oTest}->{&TEST_C} ? " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" : '') .
|
||||
' ' . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . '-test',
|
||||
' ' . $self->{strImage},
|
||||
{bSuppressStdErr => true});
|
||||
}
|
||||
}
|
||||
|
@ -379,6 +379,18 @@ sub vmCoverageC
|
||||
|
||||
push @EXPORT, qw(vmCoverageC);
|
||||
|
||||
####################################################################################################################################
|
||||
# Get vm architecture
|
||||
####################################################################################################################################
|
||||
sub vmArch
|
||||
{
|
||||
my $strVm = shift;
|
||||
|
||||
return $oyVm->{$strVm}{&VM_ARCH};
|
||||
}
|
||||
|
||||
push @EXPORT, qw(vmArch);
|
||||
|
||||
####################################################################################################################################
|
||||
# Get vm architecture bits
|
||||
####################################################################################################################################
|
||||
@ -386,7 +398,7 @@ sub vmArchBits
|
||||
{
|
||||
my $strVm = shift;
|
||||
|
||||
return ($oyVm->{$strVm}{&VM_ARCH} eq VM_ARCH_I386 ? 32 : 64);
|
||||
return (vmArch($strVm) eq VM_ARCH_I386 ? 32 : 64);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(vmArchBits);
|
||||
|
@ -1128,7 +1128,7 @@ hrnHostBuildRun(const int line, const StringId id)
|
||||
const bool isPg = strBeginsWithZ(name, "pg");
|
||||
const bool isRepo = id == hrnHostLocal.repoHost;
|
||||
const String *const container = strNewFmt("test-%u-%s", testIdx(), strZ(name));
|
||||
const String *const image = strNewFmt("pgbackrest/test:%s-test", testVm());
|
||||
const String *const image = strNewFmt("pgbackrest/test:%s-test-x86_64", testVm());
|
||||
const String *const dataPath = strNewFmt("%s/%s", testPath(), strZ(name));
|
||||
String *const option = strNewFmt(
|
||||
"-v '%s/cfg:/etc/pgbackrest:ro' -v '%s:/usr/bin/pgbackrest:ro' -v '%s:%s:ro'", strZ(dataPath), testProjectExe(),
|
||||
@ -1330,7 +1330,8 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si
|
||||
MEM_CONTEXT_PRIOR_BEGIN()
|
||||
{
|
||||
hrnHostNewP(
|
||||
HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test", testVm()), .noUpdateHosts = true);
|
||||
HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test-x86_64", testVm()),
|
||||
.noUpdateHosts = true);
|
||||
}
|
||||
MEM_CONTEXT_PRIOR_END();
|
||||
|
||||
|
@ -1951,7 +1951,7 @@ testRun(void)
|
||||
"P00 INFO: full backup size = 8KB, file total = 2",
|
||||
TEST_64BIT() ?
|
||||
(TEST_BIG_ENDIAN() ? "ead3f998dc6dbc4b444f89cd449dcb81801a21ed" : "6f7fb3cd71dbef602850d05332cdd1c8e4a64121") :
|
||||
"56d0a8a2d6b0b6dd2880c4b0221fec24e958c1a6");
|
||||
"a90290adaf2e8a36d53b0c15d277e23e1b321fdb");
|
||||
|
||||
// Make pg no longer appear to be running
|
||||
HRN_STORAGE_REMOVE(storagePgWrite(), PG_FILE_POSTMTRPID, .errorOnMissing = true);
|
||||
|
28
test/test.pl
28
test/test.pl
@ -107,6 +107,7 @@ test.pl [options]
|
||||
|
||||
VM Options:
|
||||
--vm docker container to build/test (e.g. rh8)
|
||||
--vm-arch docker container architecture
|
||||
--vm-build build Docker containers
|
||||
--vm-force force a rebuild of Docker containers
|
||||
--vm-out Show VM output (default false)
|
||||
@ -143,6 +144,7 @@ my $bHelp = false;
|
||||
my $bQuiet = false;
|
||||
my $strPgVersion = 'minimal';
|
||||
my $strVm = VM_NONE;
|
||||
my $strVmArch;
|
||||
my $bVmBuild = false;
|
||||
my $bVmForce = false;
|
||||
my $bBuildOnly = false;
|
||||
@ -183,6 +185,7 @@ GetOptions ('q|quiet' => \$bQuiet,
|
||||
'log-level-test-file=s' => \$strLogLevelTestFile,
|
||||
'no-log-timestamp' => \$bNoLogTimestamp,
|
||||
'vm=s' => \$strVm,
|
||||
'vm-arch=s' => \$strVmArch,
|
||||
'vm-out' => \$bVmOut,
|
||||
'vm-build' => \$bVmBuild,
|
||||
'vm-force' => \$bVmForce,
|
||||
@ -364,7 +367,7 @@ eval
|
||||
################################################################################################################################
|
||||
if ($bVmBuild)
|
||||
{
|
||||
containerBuild($oStorageBackRest, $strVm, $bVmForce);
|
||||
containerBuild($oStorageBackRest, $strVm, $strVmArch, $bVmForce);
|
||||
exit 0;
|
||||
}
|
||||
|
||||
@ -454,6 +457,9 @@ eval
|
||||
|
||||
# Start build container if vm is not none
|
||||
#-------------------------------------------------------------------------------------------------------------------------------
|
||||
my $strPlatform = defined($strVmArch) ? " --platform linux/${strVmArch}" : '';
|
||||
my $strImage = containerRepo() . ":${strVm}-test" . (defined($strVmArch) ? "-${strVmArch}" : '-' . hostArch());
|
||||
|
||||
if ($strVm ne VM_NONE)
|
||||
{
|
||||
my $strCCachePath = "${strTestPath}/ccache-0/${strVm}";
|
||||
@ -464,9 +470,9 @@ eval
|
||||
}
|
||||
|
||||
executeTest(
|
||||
"docker run -itd -h test-build --name=test-build" .
|
||||
"docker run${strPlatform} -itd -h test-build --name=test-build" .
|
||||
" -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath}" .
|
||||
" -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . ' ' . containerRepo() . ":${strVm}-test",
|
||||
" -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . " ${strImage}",
|
||||
{bSuppressStdErr => true});
|
||||
}
|
||||
|
||||
@ -694,8 +700,8 @@ eval
|
||||
if ($strVm ne VM_NONE)
|
||||
{
|
||||
executeTest(
|
||||
"docker run -itd -h test-build --name=test-build" .
|
||||
" -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test",
|
||||
"docker run${strPlatform} -itd -h test-build --name=test-build" .
|
||||
" -v ${strBackRestBase}:${strBackRestBase} ${strImage}",
|
||||
{bSuppressStdErr => true});
|
||||
}
|
||||
|
||||
@ -778,8 +784,8 @@ eval
|
||||
if ($strVm ne VM_NONE)
|
||||
{
|
||||
executeTest(
|
||||
"docker run -itd -h test-build --name=test-build" .
|
||||
" -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test",
|
||||
"docker run${strPlatform} -itd -h test-build --name=test-build" .
|
||||
" -v ${strBackRestBase}:${strBackRestBase} ${strImage}",
|
||||
{bSuppressStdErr => true});
|
||||
}
|
||||
|
||||
@ -958,10 +964,10 @@ eval
|
||||
if (!defined($$oyProcess[$iVmIdx]) && $iTestIdx < @{$oyTestRun})
|
||||
{
|
||||
my $oJob = new pgBackRestTest::Common::JobTest(
|
||||
$oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, $iVmMax,
|
||||
$strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, $strLogLevelTestFile, !$bNoLogTimestamp,
|
||||
$bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind, !$bNoCoverage, $bCoverageSummary,
|
||||
!$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace,
|
||||
$oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $strPlatform,
|
||||
$strImage, $iVmIdx, $iVmMax, $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest,
|
||||
$strLogLevelTestFile, !$bNoLogTimestamp, $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind,
|
||||
!$bNoCoverage, $bCoverageSummary, !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace,
|
||||
$iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax));
|
||||
$iTestIdx++;
|
||||
|
||||
|
Reference in New Issue
Block a user