From 55277357b8457e3a215bc00e8e97de523949eba6 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 20 Jul 2020 09:47:43 -0400 Subject: [PATCH] Reduce reliance on static checksums in unit tests. Testing against static checksums is valuable but it can be become burdensome when supporting multiple architectures. Reduce the number of tests we are doing against static checksums when the architecture can cause the checksum to vary. --- test/src/common/harnessTest.h | 2 + test/src/module/command/archivePushTest.c | 43 +++---- test/src/module/command/backupTest.c | 133 +++++++++++----------- 3 files changed, 89 insertions(+), 89 deletions(-) diff --git a/test/src/common/harnessTest.h b/test/src/common/harnessTest.h index d5902d7d3..ebd1c71bc 100644 --- a/test/src/common/harnessTest.h +++ b/test/src/common/harnessTest.h @@ -242,6 +242,8 @@ Macros to compare results of common data types TEST_RESULT_Z(strPtrNull(statement), strPtrNull(resultExpected), __VA_ARGS__); #define TEST_RESULT_STR_Z(statement, resultExpected, ...) \ TEST_RESULT_Z(strPtrNull(statement), resultExpected, __VA_ARGS__); +#define TEST_RESULT_STR_KEYRPL(statement, resultExpected, ...) \ + TEST_RESULT_Z(strPtrNull(statement), hrnReplaceKey(strPtr(resultExpected)), __VA_ARGS__); #define TEST_RESULT_STR_Z_KEYRPL(statement, resultExpected, ...) \ TEST_RESULT_Z(strPtrNull(statement), hrnReplaceKey(resultExpected), __VA_ARGS__); #define TEST_RESULT_Z_STR(statement, resultExpected, ...) \ diff --git a/test/src/module/command/archivePushTest.c b/test/src/module/command/archivePushTest.c index 79d4528ee..6888315a3 100644 --- a/test/src/module/command/archivePushTest.c +++ b/test/src/module/command/archivePushTest.c @@ -261,6 +261,11 @@ testRun(void) memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1)); pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer1); + // Check sha1 checksum against fixed values once to make sure they are not getting munged. After this we'll calculate them + // directly from the buffers to reduce the cost of maintaining checksums. + const char *walBuffer1Sha1 = TEST_64BIT() ? + "aae7591a1dbc58f21d0d004886075094f622e6dd" : "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a"; + storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer1); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); @@ -268,10 +273,7 @@ testRun(void) TEST_RESULT_BOOL( storageExistsP( - storageTest, - strNewFmt( - "repo/archive/test/11-1/0000000100000001/000000010000000100000001-%s.gz", - TEST_64BIT() ? "aae7591a1dbc58f21d0d004886075094f622e6dd" : "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a")), + storageTest, strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000001-%s.gz", walBuffer1Sha1)), true, "check repo for WAL file"); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again"); @@ -285,6 +287,7 @@ testRun(void) bufUsedSet(walBuffer2, bufSize(walBuffer2)); memset(bufPtr(walBuffer2), 0xFF, bufSize(walBuffer2)); pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer2); + const char *walBuffer2Sha1 = strPtr(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2))); storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer2); @@ -293,8 +296,6 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("WAL with absolute path and no pg1-path"); - const char *sha1 = TEST_64BIT() ? "755defa48a0a0872767b6dea49bdd3b64902f147" : "9c2a6ec4491a2118bcdc9b653366581d8821c982"; - argListTemp = strLstNew(); strLstAddZ(argListTemp, "--" CFGOPT_STANZA "=test"); strLstAdd(argListTemp, strNewFmt("--" CFGOPT_REPO1_PATH "=%s/repo", testPath())); @@ -309,7 +310,8 @@ testRun(void) storagePutP( storageNewWriteP( storageTest, - strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", sha1)), + strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", + walBuffer2Sha1)), BUFSTRDEF("PARTIAL")), "write WAL tmp file"); @@ -317,12 +319,13 @@ testRun(void) harnessLogResult("P00 INFO: pushed WAL file '000000010000000100000002' to the archive"); TEST_RESULT_BOOL( - storageExistsP(storageTest, strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz", sha1)), + storageExistsP( + storageTest, strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz", walBuffer2Sha1)), true, "check repo for WAL file"); TEST_RESULT_BOOL( storageExistsP( storageTest, - strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", sha1)), + strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", walBuffer2Sha1)), false, "check WAL tmp file is gone"); // Push a history file @@ -426,10 +429,7 @@ testRun(void) TEST_RESULT_BOOL( storageExistsP( - storageTest, - strNewFmt( - "repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s", - TEST_64BIT() ? "755defa48a0a0872767b6dea49bdd3b64902f147" : "9c2a6ec4491a2118bcdc9b653366581d8821c982")), + storageTest, strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)), true, "check repo for WAL file"); } @@ -587,6 +587,7 @@ testRun(void) bufUsedSet(walBuffer1, bufSize(walBuffer1)); memset(bufPtr(walBuffer1), 0xFF, bufSize(walBuffer1)); pgWalTestToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer1); + const char *walBuffer1Sha1 = strPtr(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1))); storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_xlog/000000010000000100000001")), walBuffer1); @@ -595,10 +596,7 @@ testRun(void) TEST_RESULT_BOOL( storageExistsP( - storageTest, - strNewFmt( - "repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", - TEST_64BIT() ? "ce58ec898a080e59d71dce49051843b03a497d99" : "765ee94e4f65d081f2dc3b8e77556da95bc31cbf")), + storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)), true, "check repo for WAL file"); // Direct tests of the async function @@ -660,10 +658,7 @@ testRun(void) TEST_RESULT_BOOL( storageExistsP( - storageTest, - strNewFmt( - "repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", - TEST_64BIT() ? "ce58ec898a080e59d71dce49051843b03a497d99" : "765ee94e4f65d081f2dc3b8e77556da95bc31cbf")), + storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)), true, "check repo for WAL 1 file"); TEST_RESULT_STR_Z( @@ -675,6 +670,7 @@ testRun(void) bufUsedSet(walBuffer2, bufSize(walBuffer2)); memset(bufPtr(walBuffer2), 0x0C, bufSize(walBuffer2)); pgWalTestToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer2); + const char *walBuffer2Sha1 = strPtr(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2))); storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_xlog/000000010000000100000002")), walBuffer2); @@ -689,10 +685,7 @@ testRun(void) TEST_RESULT_BOOL( storageExistsP( - storageTest, - strNewFmt( - "repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", - TEST_64BIT() ? "251820f85b31554bbf33061fce7dabf18c16afe0" : "ae8cb1fe45d40bc364df6bc6e8803774f0e3eaeb")), + storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)), true, "check repo for WAL 2 file"); TEST_RESULT_STR_Z( diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index d8dd5d2cf..e743ad9ef 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -2269,6 +2269,7 @@ testRun(void) bufUsedSet(relation, bufSize(relation)); storagePutP(storageNewWriteP(storagePgWrite(), STRDEF(PG_PATH_BASE "/1/3"), .timeModified = backupTimeStart), relation); + const char *rel1_3Sha1 = strPtr(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, relation))); // File with bad page checksum relation = bufNew(PG_PAGE_SIZE_DEFAULT * 3); @@ -2279,6 +2280,7 @@ testRun(void) bufUsedSet(relation, bufSize(relation)); storagePutP(storageNewWriteP(storagePgWrite(), STRDEF(PG_PATH_BASE "/1/4"), .timeModified = backupTimeStart), relation); + const char *rel1_4Sha1 = strPtr(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, relation))); // Add a tablespace storagePathCreateP(storagePgWrite(), STRDEF(PG_PATH_PGTBLSPC)); @@ -2327,71 +2329,74 @@ testRun(void) "P00 INFO: check archive for segment(s) 0000000105DB5DE000000000:0000000105DB5DE000000002\n" "P00 INFO: new backup label = 20191027-181320F"); - TEST_RESULT_STR_Z_KEYRPL( + TEST_RESULT_STR_KEYRPL( testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/20191027-181320F")), - "pg_data {path}\n" - "pg_data/PG_VERSION.gz {file, s=2}\n" - "pg_data/backup_label.gz {file, s=17}\n" - "pg_data/base {path}\n" - "pg_data/base/1 {path}\n" - "pg_data/base/1/1.gz {file, s=8192}\n" - "pg_data/base/1/2.gz {file, s=8193}\n" - "pg_data/base/1/3.gz {file, s=32768}\n" - "pg_data/base/1/4.gz {file, s=24576}\n" - "pg_data/global {path}\n" - "pg_data/global/pg_control.gz {file, s=8192}\n" - "pg_data/pg_tblspc {path}\n" - "pg_data/pg_wal {path}\n" - "pg_data/pg_wal/0000000105DB5DE000000000.gz {file, s=1048576}\n" - "pg_data/pg_wal/0000000105DB5DE000000001.gz {file, s=1048576}\n" - "pg_data/pg_wal/0000000105DB5DE000000002.gz {file, s=1048576}\n" - "pg_data/postgresql.conf.gz {file, s=11}\n" - "pg_tblspc {path}\n" - "pg_tblspc/32768 {path}\n" - "pg_tblspc/32768/PG_11_201809051 {path}\n" - "pg_tblspc/32768/PG_11_201809051/1 {path}\n" - "pg_tblspc/32768/PG_11_201809051/1/5.gz {file, s=0}\n" - "--------\n" - "[backup:target]\n" - "pg_data={\"path\":\"{[path]}/pg1\",\"type\":\"path\"}\n" - "pg_tblspc/32768={\"path\":\"../../pg1-tblspc/32768\",\"tablespace-id\":\"32768\",\"tablespace-name\":\"tblspc32768\",\"type\":\"link\"}\n" - "\n" - "[target:file]\n" - "pg_data/PG_VERSION={\"checksum\":\"17ba0791499db908433b80f37c5fbc89b870084b\",\"size\":2" - ",\"timestamp\":1572200000}\n" - "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17" - ",\"timestamp\":1572200002}\n" - "pg_data/base/1/1={\"checksum\":\"0631457264ff7f8d5fb1edc2c0211992a67c73e6\",\"checksum-page\":true" - ",\"master\":false,\"size\":8192,\"timestamp\":1572200000}\n" - "pg_data/base/1/2={\"checksum\":\"8beb58e08394fe665fb04a17b4003faa3802760b\",\"checksum-page\":false" - ",\"master\":false,\"size\":8193,\"timestamp\":1572200000}\n" - "pg_data/base/1/3={\"checksum\":\"73e537a445ad34eab4b292ac6aa07b8ce14e8421\",\"checksum-page\":false" - ",\"checksum-page-error\":[0,[2,3]],\"master\":false,\"size\":32768,\"timestamp\":1572200000}\n" - "pg_data/base/1/4={\"checksum\":\"ba233be7198b3115f0480fa5274448f2a2fc2af1\",\"checksum-page\":false" - ",\"checksum-page-error\":[1],\"master\":false,\"size\":24576,\"timestamp\":1572200000}\n" - "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572200000}\n" - "pg_data/pg_wal/0000000105DB5DE000000000={\"size\":1048576,\"timestamp\":1572200002}\n" - "pg_data/pg_wal/0000000105DB5DE000000001={\"size\":1048576,\"timestamp\":1572200002}\n" - "pg_data/pg_wal/0000000105DB5DE000000002={\"size\":1048576,\"timestamp\":1572200002}\n" - "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11" - ",\"timestamp\":1570000000}\n" - "pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"master\":false,\"size\":0" - ",\"timestamp\":1572200000}\n" - "\n" - "[target:link]\n" - "pg_data/pg_tblspc/32768={\"destination\":\"../../pg1-tblspc/32768\"}\n" - "\n" - "[target:path]\n" - "pg_data={}\n" - "pg_data/base={}\n" - "pg_data/base/1={}\n" - "pg_data/global={}\n" - "pg_data/pg_tblspc={}\n" - "pg_data/pg_wal={}\n" - "pg_tblspc={}\n" - "pg_tblspc/32768={}\n" - "pg_tblspc/32768/PG_11_201809051={}\n" - "pg_tblspc/32768/PG_11_201809051/1={}\n", + strNewFmt( + "pg_data {path}\n" + "pg_data/PG_VERSION.gz {file, s=2}\n" + "pg_data/backup_label.gz {file, s=17}\n" + "pg_data/base {path}\n" + "pg_data/base/1 {path}\n" + "pg_data/base/1/1.gz {file, s=8192}\n" + "pg_data/base/1/2.gz {file, s=8193}\n" + "pg_data/base/1/3.gz {file, s=32768}\n" + "pg_data/base/1/4.gz {file, s=24576}\n" + "pg_data/global {path}\n" + "pg_data/global/pg_control.gz {file, s=8192}\n" + "pg_data/pg_tblspc {path}\n" + "pg_data/pg_wal {path}\n" + "pg_data/pg_wal/0000000105DB5DE000000000.gz {file, s=1048576}\n" + "pg_data/pg_wal/0000000105DB5DE000000001.gz {file, s=1048576}\n" + "pg_data/pg_wal/0000000105DB5DE000000002.gz {file, s=1048576}\n" + "pg_data/postgresql.conf.gz {file, s=11}\n" + "pg_tblspc {path}\n" + "pg_tblspc/32768 {path}\n" + "pg_tblspc/32768/PG_11_201809051 {path}\n" + "pg_tblspc/32768/PG_11_201809051/1 {path}\n" + "pg_tblspc/32768/PG_11_201809051/1/5.gz {file, s=0}\n" + "--------\n" + "[backup:target]\n" + "pg_data={\"path\":\"{[path]}/pg1\",\"type\":\"path\"}\n" + "pg_tblspc/32768={\"path\":\"../../pg1-tblspc/32768\",\"tablespace-id\":\"32768\"" + ",\"tablespace-name\":\"tblspc32768\",\"type\":\"link\"}\n" + "\n" + "[target:file]\n" + "pg_data/PG_VERSION={\"checksum\":\"17ba0791499db908433b80f37c5fbc89b870084b\",\"size\":2" + ",\"timestamp\":1572200000}\n" + "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17" + ",\"timestamp\":1572200002}\n" + "pg_data/base/1/1={\"checksum\":\"0631457264ff7f8d5fb1edc2c0211992a67c73e6\",\"checksum-page\":true" + ",\"master\":false,\"size\":8192,\"timestamp\":1572200000}\n" + "pg_data/base/1/2={\"checksum\":\"8beb58e08394fe665fb04a17b4003faa3802760b\",\"checksum-page\":false" + ",\"master\":false,\"size\":8193,\"timestamp\":1572200000}\n" + "pg_data/base/1/3={\"checksum\":\"%s\",\"checksum-page\":false,\"checksum-page-error\":[0,[2,3]]" + ",\"master\":false,\"size\":32768,\"timestamp\":1572200000}\n" + "pg_data/base/1/4={\"checksum\":\"%s\",\"checksum-page\":false,\"checksum-page-error\":[1],\"master\":false" + ",\"size\":24576,\"timestamp\":1572200000}\n" + "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572200000}\n" + "pg_data/pg_wal/0000000105DB5DE000000000={\"size\":1048576,\"timestamp\":1572200002}\n" + "pg_data/pg_wal/0000000105DB5DE000000001={\"size\":1048576,\"timestamp\":1572200002}\n" + "pg_data/pg_wal/0000000105DB5DE000000002={\"size\":1048576,\"timestamp\":1572200002}\n" + "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11" + ",\"timestamp\":1570000000}\n" + "pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"master\":false,\"size\":0" + ",\"timestamp\":1572200000}\n" + "\n" + "[target:link]\n" + "pg_data/pg_tblspc/32768={\"destination\":\"../../pg1-tblspc/32768\"}\n" + "\n" + "[target:path]\n" + "pg_data={}\n" + "pg_data/base={}\n" + "pg_data/base/1={}\n" + "pg_data/global={}\n" + "pg_data/pg_tblspc={}\n" + "pg_data/pg_wal={}\n" + "pg_tblspc={}\n" + "pg_tblspc/32768={}\n" + "pg_tblspc/32768/PG_11_201809051={}\n" + "pg_tblspc/32768/PG_11_201809051/1={}\n", + rel1_3Sha1, rel1_4Sha1), "compare file list"); // Remove test files