From 39dddbb6bc7d296fe25a7210468b2f51ea6b1816 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 9 May 2022 12:48:19 -0400 Subject: [PATCH] Add limited CI for ppc64le/s390x using emulation. The emulation is so slow that running all the unit tests would be too expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to test on one big-endian architecture to be sure that checksums are correct. Update checksums in the tests where they had gotten out of date since the last time we were testing on s390x. Also use a different test in command/archivePushTest to show the name of the file when a checksum does not match to aid in debugging. The command/archive-push test was updated but not included because there is also a permissions issue, which looks to be the same as what we see on MacOS/FreedBSD. Hopefully we'll be able to fix all of those at the same time. --- .github/workflows/test.yml | 39 +++++++++++++++++++++++ test/src/module/command/archivePushTest.c | 17 +++++----- test/src/module/command/backupTest.c | 8 ++--- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 96ab5aa12..0d2726acd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,6 +55,45 @@ jobs: - name: Run Test run: cd ${HOME?} && ${GITHUB_WORKSPACE?}/pgbackrest/test/ci.pl ${{matrix.param}} --param=build-max=2 + # Basic tests on other architectures using emulation. The emulation is so slow that running all the unit tests would be too + # expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to + # test on one big-endian architecture to be sure that checksums are correct. + arch: + runs-on: ubuntu-20.04 + + strategy: + matrix: + include: + - arch: ppc64le + distro: ubuntu20.04 + - arch: s390x + distro: ubuntu20.04 + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + with: + path: pgbackrest + + - name: Run Test + uses: uraimo/run-on-arch-action@v2.1.0 + id: runcmd + with: + arch: ${{matrix.arch}} + distro: ${{matrix.distro}} + + # Cache builds + githubToken: ${{github.token}} + + install: | + apt-get update + DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev + + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE?}/pgbackrest + ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=none --no-gen --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup + ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=none --no-gen --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface + codeql: runs-on: ubuntu-latest diff --git a/test/src/module/command/archivePushTest.c b/test/src/module/command/archivePushTest.c index 6f89942c0..0c41090b9 100644 --- a/test/src/module/command/archivePushTest.c +++ b/test/src/module/command/archivePushTest.c @@ -349,19 +349,20 @@ testRun(void) memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1)); hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11}, walBuffer1); - // Check sha1 checksum against fixed values once to make sure they are not getting munged. After this we'll calculate them - // directly from the buffers to reduce the cost of maintaining checksums. - walBuffer1Sha1 = TEST_64BIT() ? - (TEST_BIG_ENDIAN() ? "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" : "858a9ef24b79468eb2a61543b58140addfede0fc") : - "044ec0576dc4e59d460aa3a8ac796ba4874ddff3"; - HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive"); - TEST_STORAGE_EXISTS( - storageRepoIdxWrite(0), zNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1), + // Check sha1 checksum against fixed values once to make sure they are not getting munged. After this we'll calculate them + // directly from the buffers to reduce the cost of maintaining checksums. + TEST_STORAGE_LIST( + storageRepoIdx(0), STORAGE_REPO_ARCHIVE "/11-1/0000000100000001", + zNewFmt( + "000000010000000100000001-%s.gz\n", + TEST_64BIT() ? + (TEST_BIG_ENDIAN() ? "4dc9df63290935f68f43b7d02005716a98800ce0" : "858a9ef24b79468eb2a61543b58140addfede0fc") : + "044ec0576dc4e59d460aa3a8ac796ba4874ddff3"), .comment = "check repo for WAL file"); // No warning emitted re WAL file already existing with the same checksum due to --no-archive-mode-check diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index f2d98ca0e..514f62852 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -2006,7 +2006,7 @@ testRun(void) "P00 INFO: new backup label = [FULL-1]\n" "P00 INFO: full backup size = 8KB, file total = 2", TEST_64BIT() ? - (TEST_BIG_ENDIAN() ? "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" : "b7ec43e4646f5d06c95881df0c572630a1221377") : + (TEST_BIG_ENDIAN() ? "ec84602c8b4f62bd0ef10bd3dfcb04c3b3ce4a35" : "b7ec43e4646f5d06c95881df0c572630a1221377") : "f21ff9abdcd1ec2f600d4ee8e5792c9b61eb2e37"); // Make pg no longer appear to be running @@ -2835,6 +2835,7 @@ testRun(void) *(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x01)) = (PageHeaderData){.pd_upper = 0xFF}; HRN_STORAGE_PUT(storagePgWrite(), PG_PATH_BASE "/1/2", relation, .timeModified = backupTimeStart); + const char *rel1_2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, relation))); // File with bad page checksums relation = bufNew(PG_PAGE_SIZE_DEFAULT * 5); @@ -2954,8 +2955,7 @@ testRun(void) ",\"timestamp\":1572200002}\n" "pg_data/base/1/1={\"checksum\":\"0631457264ff7f8d5fb1edc2c0211992a67c73e6\",\"checksum-page\":true" ",\"size\":8192,\"timestamp\":1572200000}\n" - "pg_data/base/1/2={\"checksum\":\"2deafa7ae60279a54a09422b985a8025f5e125fb\",\"checksum-page\":false" - ",\"size\":8704,\"timestamp\":1572200000}\n" + "pg_data/base/1/2={\"checksum\":\"%s\",\"checksum-page\":false,\"size\":8704,\"timestamp\":1572200000}\n" "pg_data/base/1/3={\"checksum\":\"%s\",\"checksum-page\":false,\"checksum-page-error\":[0,[2,4]]" ",\"size\":40960,\"timestamp\":1572200000}\n" "pg_data/base/1/4={\"checksum\":\"%s\",\"checksum-page\":false,\"checksum-page-error\":[1],\"size\":24576" @@ -2984,7 +2984,7 @@ testRun(void) "pg_tblspc/32768={}\n" "pg_tblspc/32768/PG_11_201809051={}\n" "pg_tblspc/32768/PG_11_201809051/1={}\n", - rel1_3Sha1, rel1_4Sha1), + rel1_2Sha1, rel1_3Sha1, rel1_4Sha1), "compare file list"); // Remove test files