From bdff1a749262fe944e4533cd2fd7617226a538a8 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Tue, 12 May 2020 17:47:22 +0300 Subject: [PATCH 01/14] Add latest release badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e908f273..ad098872 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) +[![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest) # pg_probackup From e95919228588c280f3566c371b3ad285c9ba5f93 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 13:43:13 +0300 Subject: [PATCH 02/14] replace rmtree with pgut_rmtree --- src/archive.c | 2 +- src/delete.c | 2 +- src/pg_probackup.h | 2 + src/utils/pgut.c | 140 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 2 deletions(-) diff --git a/src/archive.c b/src/archive.c index cbe57229..c885184d 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1109,7 +1109,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, { /* discard prefetch */ // n_fetched = 0; - rmtree(prefetch_dir, false); + pgut_rmtree(prefetch_dir, false, false); } } else diff --git a/src/delete.c b/src/delete.c index 5da60da9..b5e6fb68 100644 --- a/src/delete.c +++ b/src/delete.c @@ -986,7 +986,7 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - rmtree(arclog_path, false); + pgut_rmtree(arclog_path, false, true); /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3ce98c4a..a2d65b88 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1030,6 +1030,8 @@ extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonL extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); +extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); + /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) #define FILE_MISSING (-1) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index b7599816..6d996f47 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -43,6 +43,9 @@ static void on_interrupt(void); static void on_cleanup(void); static pqsigfunc oldhandler = NULL; +static char ** pgut_pgfnames(const char *path, bool strict); +static void pgut_pgfnames_cleanup(char **filenames); + void discard_response(PGconn *conn); void @@ -1062,3 +1065,140 @@ discard_response(PGconn *conn) PQclear(res); } while (res); } + +/* + * pgfnames + * + * return a list of the names of objects in the argument directory. Caller + * must call pgfnames_cleanup later to free the memory allocated by this + * function. + */ +char ** +pgut_pgfnames(const char *path, bool strict) +{ + DIR *dir; + struct dirent *file; + char **filenames; + int numnames = 0; + int fnsize = 200; /* enough for many small dbs */ + + dir = opendir(path); + if (dir == NULL) + { + elog(strict ? ERROR : WARNING, "could not open directory \"%s\": %m", path); + return NULL; + } + + filenames = (char **) palloc(fnsize * sizeof(char *)); + + while (errno = 0, (file = readdir(dir)) != NULL) + { + if (strcmp(file->d_name, ".") != 0 && strcmp(file->d_name, "..") != 0) + { + if (numnames + 1 >= fnsize) + { + fnsize *= 2; + filenames = (char **) repalloc(filenames, + fnsize * sizeof(char *)); + } + filenames[numnames++] = pstrdup(file->d_name); + } + } + + if (errno) + { + elog(strict ? ERROR : WARNING, "could not read directory \"%s\": %m", path); + return NULL; + } + + filenames[numnames] = NULL; + + if (closedir(dir)) + { + elog(strict ? ERROR : WARNING, "could not close directory \"%s\": %m", path); + return NULL; + } + + return filenames; +} + +/* + * pgfnames_cleanup + * + * deallocate memory used for filenames + */ +void +pgut_pgfnames_cleanup(char **filenames) +{ + char **fn; + + for (fn = filenames; *fn; fn++) + pfree(*fn); + + pfree(filenames); +} + +/* Shamelessly stolen from commom/rmtree.c */ +bool +pgut_rmtree(const char *path, bool rmtopdir, bool strict) +{ + bool result = true; + char pathbuf[MAXPGPATH]; + char **filenames; + char **filename; + struct stat statbuf; + + /* + * we copy all the names out of the directory before we start modifying + * it. + */ + filenames = pgut_pgfnames(path, strict); + + if (filenames == NULL) + return false; + + /* now we have the names we can start removing things */ + for (filename = filenames; *filename; filename++) + { + snprintf(pathbuf, MAXPGPATH, "%s/%s", path, *filename); + + if (lstat(pathbuf, &statbuf) != 0) + { + elog(strict ? ERROR : WARNING, "could not stat file or directory \"%s\": %m", pathbuf); + result = false; + break; + } + + if (S_ISDIR(statbuf.st_mode)) + { + /* call ourselves recursively for a directory */ + if (!pgut_rmtree(pathbuf, true, strict)) + { + result = false; + break; + } + } + else + { + if (unlink(pathbuf) != 0) + { + elog(strict ? ERROR : WARNING, "could not remove file or directory \"%s\": %m", pathbuf); + result = false; + break; + } + } + } + + if (rmtopdir) + { + if (rmdir(path) != 0) + { + elog(strict ? ERROR : WARNING, "could not remove file or directory \"%s\": %m", path); + result = false; + } + } + + pgut_pgfnames_cleanup(filenames); + + return result; +} From a4e9eff91b58173cd404e3d295f3e9c2e82424c4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 13:44:22 +0300 Subject: [PATCH 03/14] tests: added tests.exclude.ExcludeTest.test_exclude_log_dir_1 and tests.exclude.ExcludeTest.test_exclude_log_dir --- tests/exclude.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/exclude.py b/tests/exclude.py index 0402c978..c9efe22a 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -153,3 +153,85 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exclude_log_dir(self): + """ + check that by default 'log' and 'pg_log' directories are not backed up + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + log_dir = node.safe_psql( + 'postgres', + 'show log_directory').rstrip() + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + # check that PGDATA/log or PGDATA/pg_log do not exists + path = os.path.join(node.data_dir, log_dir) + log_file = os.path.join(path, 'postgresql.log') + self.assertTrue(os.path.exists(path)) + self.assertFalse(os.path.exists(log_file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exclude_log_dir_1(self): + """ + check that "--backup-pg-log" works correctly + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + log_dir = node.safe_psql( + 'postgres', + 'show log_directory').rstrip() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream', '--backup-pg-log']) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + # check that PGDATA/log or PGDATA/pg_log do not exists + path = os.path.join(node.data_dir, log_dir) + log_file = os.path.join(path, 'postgresql.log') + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.exists(log_file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 61b476bb312110ec2621207d5d0114f8df8e7e27 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 15 Apr 2020 19:17:21 +0300 Subject: [PATCH 04/14] Correct expected error message in the test_ptrack_pg_resetxlog --- tests/ptrack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 309a3c44..6ca8d93c 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3841,7 +3841,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - @unittest.expectedFailure + # @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): fname = self.id().split('.')[3] node = self.make_simple_node( @@ -3946,7 +3946,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) except ProbackupException as e: self.assertIn( - 'Insert error message', + 'ERROR: LSN from ptrack_control 0/0 differs from STOP LSN of previous backup', e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) From 583ffaaa307e0861b71ab560140bc43f65ef4fe1 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 22 Apr 2020 19:30:39 +0300 Subject: [PATCH 05/14] Teach pg_probackup to work with ptrack 2.1 --- doc/pgprobackup.xml | 10 +++++----- src/backup.c | 2 +- src/ptrack.c | 4 +++- tests/helpers/ptrack_helpers.py | 3 ++- tests/ptrack.py | 13 +++++++------ 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 58d42749..a2161f0c 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1168,12 +1168,12 @@ CREATE EXTENSION ptrack; - To enable tracking page updates, set ptrack_map_size + To enable tracking page updates, set ptrack.map_size parameter to a positive integer and restart the server. For optimal performance, it is recommended to set - ptrack_map_size to + ptrack.map_size to N / 1024, where N is the size of the PostgreSQL cluster, in MB. If you set this @@ -1181,7 +1181,7 @@ CREATE EXTENSION ptrack; together, which leads to false-positive results when tracking changed blocks and increases the incremental backup size as unchanged blocks can also be copied into the incremental backup. - Setting ptrack_map_size to a higher value + Setting ptrack.map_size to a higher value does not affect PTRACK operation. The maximum allowed value is 1024. @@ -1201,11 +1201,11 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; - If you change the ptrack_map_size parameter value, + If you change the ptrack.map_size parameter value, the previously created PTRACK map file is cleared, and tracking newly changed blocks starts from scratch. Thus, you have to retake a full backup before taking incremental PTRACK backups after - changing ptrack_map_size. + changing ptrack.map_size. diff --git a/src/backup.c b/src/backup.c index 96b47916..4ec515a1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -436,7 +436,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* * Build the page map from ptrack information. */ - if (nodeInfo->ptrack_version_num == 20) + if (nodeInfo->ptrack_version_num >= 20) make_pagemap_from_ptrack_2(backup_files_list, backup_conn, nodeInfo->ptrack_schema, prev_backup_start_lsn); diff --git a/src/ptrack.c b/src/ptrack.c index ee39d23b..a267f3f4 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -204,6 +204,8 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) nodeInfo->ptrack_version_num = 17; else if (strcmp(ptrack_version_str, "2.0") == 0) nodeInfo->ptrack_version_num = 20; + else if (strcmp(ptrack_version_str, "2.1") == 0) + nodeInfo->ptrack_version_num = 21; else elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", ptrack_version_str); @@ -572,7 +574,7 @@ pg_ptrack_enable2(PGconn *backup_conn) { PGresult *res_db; - res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0) { diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 52a1ffda..eec57a3b 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -366,7 +366,8 @@ class ProbackupTest(object): if ptrack_enable: if node.major_version > 11: - options['ptrack_map_size'] = '128MB' + options['ptrack.map_size'] = '128' + options['shared_preload_libraries'] = 'ptrack' else: options['ptrack_enable'] = 'on' diff --git a/tests/ptrack.py b/tests/ptrack.py index 6ca8d93c..4678708b 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -269,7 +269,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'}) + 'checkpoint_timeout': '30s', + 'shared_preload_libraries': 'ptrack'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -336,16 +337,16 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # DISABLE PTRACK if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack_map_size to 0") + node.safe_psql('postgres', "alter system set ptrack.map_size to 0") else: node.safe_psql('postgres', "alter system set ptrack_enable to off") - node.stop() node.slow_start() # ENABLE PTRACK if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack_map_size to '128MB'") + node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") + node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") else: node.safe_psql('postgres', "alter system set ptrack_enable to on") node.stop() @@ -4054,7 +4055,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): 'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map), log_content) - self.set_auto_conf(node, {'ptrack_map_size': '0'}) + self.set_auto_conf(node, {'ptrack.map_size': '0'}) node.slow_start() @@ -4082,7 +4083,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.stop(['-m', 'immediate', '-D', node.data_dir]) - self.set_auto_conf(node, {'ptrack_map_size': '32'}) + self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) node.slow_start() From 667a80edfb9cf5278d6791478ab8035b0529b411 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 29 Apr 2020 20:55:39 +0300 Subject: [PATCH 06/14] Adopt ptrack 2.1 public API changes --- src/backup.c | 14 ++++----- src/pg_probackup.h | 10 ++++--- src/ptrack.c | 75 ++++++++++++++++++++++++---------------------- tests/ptrack.py | 2 +- 4 files changed, 52 insertions(+), 49 deletions(-) diff --git a/src/backup.c b/src/backup.c index 4ec515a1..1bcae2c0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -438,8 +438,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) */ if (nodeInfo->ptrack_version_num >= 20) make_pagemap_from_ptrack_2(backup_files_list, backup_conn, - nodeInfo->ptrack_schema, - prev_backup_start_lsn); + nodeInfo->ptrack_schema, + nodeInfo->ptrack_version_num, + prev_backup_start_lsn); else if (nodeInfo->ptrack_version_num == 15 || nodeInfo->ptrack_version_num == 16 || nodeInfo->ptrack_version_num == 17) @@ -884,15 +885,10 @@ do_backup(time_t start_time, bool no_validate, #endif get_ptrack_version(backup_conn, &nodeInfo); -// elog(WARNING, "ptrack_version_num %d", ptrack_version_num); + // elog(WARNING, "ptrack_version_num %d", ptrack_version_num); if (nodeInfo.ptrack_version_num > 0) - { - if (nodeInfo.ptrack_version_num >= 20) - nodeInfo.is_ptrack_enable = pg_ptrack_enable2(backup_conn); - else - nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn); - } + nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num); if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 93af776c..79de41b8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -999,11 +999,12 @@ extern void parse_filelist_filenames(parray *files, const char *root); /* in ptrack.c */ extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn); extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, - const char *ptrack_schema, XLogRecPtr lsn); + const char *ptrack_schema, + int ptrack_version_num, + XLogRecPtr lsn); extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num); extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern bool pg_ptrack_enable(PGconn *backup_conn); -extern bool pg_ptrack_enable2(PGconn *backup_conn); +extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num); extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn); extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, @@ -1011,7 +1012,8 @@ extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, size_t *result_size, PGconn *backup_conn); extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn); +extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, + int ptrack_version_num, XLogRecPtr lsn); /* FIO */ extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, diff --git a/src/ptrack.c b/src/ptrack.c index a267f3f4..035bad13 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -217,19 +217,29 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) * Check if ptrack is enabled in target instance */ bool -pg_ptrack_enable(PGconn *backup_conn) +pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) { PGresult *res_db; + bool result = false; - res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + if (ptrack_version_num < 20) { - PQclear(res_db); - return false; + res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0; } + else if (ptrack_version_num == 20) + { + res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + } + else + { + res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + } + PQclear(res_db); - return true; + return result; } @@ -459,7 +469,11 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) { char query[128]; - sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + if (nodeInfo->ptrack_version_num == 20) + sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); + res = pgut_execute(backup_conn, query, 0, NULL); } @@ -526,7 +540,10 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + if (ptrack_version_num == 20) + sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, @@ -566,30 +583,12 @@ pg_ptrack_get_block(ConnectionArgs *arguments, * ---------------------------- */ -/* - * Check if ptrack is enabled in target instance - */ -bool -pg_ptrack_enable2(PGconn *backup_conn) -{ - PGresult *res_db; - - res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0) - { - PQclear(res_db); - return false; - } - PQclear(res_db); - return true; -} - /* * Fetch a list of changed files with their ptrack maps. */ parray * -pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn) +pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, + int ptrack_version_num, XLogRecPtr lsn) { PGresult *res; char lsn_buf[17 + 1]; @@ -604,8 +603,12 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", - ptrack_schema); + if (ptrack_version_num == 20) + sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); + else + sprintf(query, "SELECT path, pagemap FROM %s.ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); res = pgut_execute(backup_conn, query, 1, (const char **) params); pfree(params[0]); @@ -647,16 +650,18 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec */ void make_pagemap_from_ptrack_2(parray *files, - PGconn *backup_conn, - const char *ptrack_schema, - XLogRecPtr lsn) + PGconn *backup_conn, + const char *ptrack_schema, + int ptrack_version_num, + XLogRecPtr lsn) { parray *filemaps; int file_i = 0; page_map_entry *dummy_map = NULL; /* Receive all available ptrack bitmaps at once */ - filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, lsn); + filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, + ptrack_version_num, lsn); if (filemaps != NULL) parray_qsort(filemaps, pgFileMapComparePath); diff --git a/tests/ptrack.py b/tests/ptrack.py index 4678708b..9f9dbac5 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4052,7 +4052,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): log_content = f.read() self.assertIn( - 'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map), + 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), log_content) self.set_auto_conf(node, {'ptrack.map_size': '0'}) From 75f8a1a5852bf4a48545b3bdf5c93b6615497788 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 6 May 2020 15:58:16 +0300 Subject: [PATCH 07/14] Skip test_ptrack_get_block for ptrack 2.* --- tests/ptrack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ptrack.py b/tests/ptrack.py index 9f9dbac5..db158081 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -631,6 +631,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.slow_start() if node.major_version >= 12: + self.skipTest("skip --- we do not need ptrack_get_block for ptrack 2.*") node.safe_psql( "postgres", "CREATE EXTENSION ptrack") From aa336b4e5041a7d5d0ceb7c4e1978b66a859b315 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Thu, 7 May 2020 14:47:20 +0300 Subject: [PATCH 08/14] Change a way of setting up python environment --- travis/Dockerfile.in | 6 +++++- travis/run_tests.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index 68f5ffe2..a3c858ee 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -2,7 +2,11 @@ FROM ololobus/postgres-dev:stretch USER root RUN apt-get update -RUN apt-get -yq install python python-pip python-virtualenv +RUN apt-get -yq install python python-pip + +# RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +# RUN python2 get-pip.py +RUN python2 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 95cb646c..1bb3a6fd 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -60,7 +60,7 @@ make USE_PGXS=1 top_srcdir=$PG_SRC install # Setup python environment echo "############### Setting up python env:" -virtualenv pyenv +python2 -m virtualenv pyenv source pyenv/bin/activate pip install testgres==1.8.2 From 23497fb78d2a310e94a90f24af6921d210174d0c Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 8 May 2020 17:42:22 +0300 Subject: [PATCH 09/14] Check for ptrack.map_size = -1 for ptrack >= 2.1 --- src/ptrack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ptrack.c b/src/ptrack.c index 035bad13..06c54308 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -235,7 +235,8 @@ pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) else { res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0; } PQclear(res_db); From 8a6b5af2972f98704c2f054feb9251ba0581c8e7 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sat, 9 May 2020 15:19:36 +0300 Subject: [PATCH 10/14] Use pg_ptrack_get_block() only with ptrack <2.0.0 --- src/data.c | 17 ++++++++--------- src/ptrack.c | 5 +++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/data.c b/src/data.c index 1270de4f..83e95c86 100644 --- a/src/data.c +++ b/src/data.c @@ -408,15 +408,12 @@ prepare_page(ConnectionArgs *conn_arg, } } - /* Get page via ptrack interface from PostgreSQL shared buffer. - * We do this in following cases: - * 1. PTRACK backup of 1.x versions - * 2. During backup, regardless of backup mode, of PostgreSQL instance - * with ptrack support we encountered invalid page. + /* + * Get page via ptrack interface from PostgreSQL shared buffer. + * We do this only in the cases of PTRACK 1.x versions backup */ - if ((backup_mode == BACKUP_MODE_DIFF_PTRACK + if (backup_mode == BACKUP_MODE_DIFF_PTRACK && (ptrack_version_num >= 15 && ptrack_version_num < 20)) - || !page_is_valid) { int rc = 0; size_t page_size = 0; @@ -440,7 +437,8 @@ prepare_page(ConnectionArgs *conn_arg, memcpy(page, ptrack_page, BLCKSZ); pg_free(ptrack_page); - /* UPD: It apprears that is possible to get zeroed page or page with invalid header + /* + * UPD: It apprears that is possible to get zeroed page or page with invalid header * from shared buffer. * Note, that getting page with wrong checksumm from shared buffer is * acceptable. @@ -462,7 +460,8 @@ prepare_page(ConnectionArgs *conn_arg, from_fullpath, blknum, errormsg); } - /* We must set checksum here, because it is outdated + /* + * We must set checksum here, because it is outdated * in the block recieved from shared buffers. */ if (checksum_version) diff --git a/src/ptrack.c b/src/ptrack.c index 06c54308..0cb59960 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -526,7 +526,7 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (arguments->cancel_conn == NULL) arguments->cancel_conn = PQgetCancel(arguments->conn); - //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + // elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); if (ptrack_version_num < 20) res = pgut_execute_parallel(arguments->conn, @@ -544,7 +544,8 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (ptrack_version_num == 20) sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); else - sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()"); + // sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, From 0b1ae536be9dee9fe353a78587af2d2e6b6683ba Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sun, 10 May 2020 14:30:29 +0300 Subject: [PATCH 11/14] Fix (?) test_ptrack_uncommitted_xact --- tests/ptrack.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index db158081..7d304d52 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -413,7 +413,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) + # TODO: what's the point in taking pgdata content, then taking + # backup, and the trying to compare those two? Backup issues a + # checkpoint, so it will modify pgdata with close to 100% chance. + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) self.backup_node( backup_dir, 'node', node, backup_type='ptrack', @@ -427,8 +431,9 @@ class PtrackTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node_restored, node_restored.data_dir, options=["-j", "4"]) - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.set_auto_conf( node_restored, {'port': node_restored.port}) @@ -436,7 +441,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node_restored.slow_start() # Physical comparison - self.compare_pgdata(pgdata, pgdata_restored) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) From 4609347796105991b82d79f0ccc6cc706e8936a4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 16:47:15 +0300 Subject: [PATCH 12/14] DOC: minor change of wording in archive-push and archive-get --- doc/pgprobackup.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index a2161f0c..91ec1253 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -4055,10 +4055,10 @@ pg_probackup archive-push -B backup_dir --instance - To speed up archiving, you can specify the option - to run archive-push on multiple threads. - If you provide the option, WAL files - will be copied in batches of the specified size. + To speed up archiving, you can specify the option + to copy WAL segments in batches of the specified size. + If option is used, then you can also specify + the option to copy the batch of WAL segments on multiple threads. WAL segments copied to the archive are synced to disk unless @@ -4096,10 +4096,10 @@ pg_probackup archive-get -B backup_dir --instance - To speed up recovery, you can specify the option - to run archive-get on multiple threads. - If you provide the option, WAL segments - will be copied in batches of the specified size. + To speed up recovery, you can specify the option + to copy WAL segments in batches of the specified size. + If option is used, then you can also specify + the option to copy the batch of WAL segments on multiple threads. From b799e6d9507a1eed513affa9fa226e59c4c9068e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 19:04:45 +0300 Subject: [PATCH 13/14] minor spelling fix --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 1bcae2c0..2e67c3e3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -192,7 +192,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) { /* try to setup multi-timeline backup chain */ elog(WARNING, "Valid backup on current timeline %u is not found, " - "try to look up on previous timelines", + "trying to look up on previous timelines", current.tli); tli_list = catalog_get_timelines(&instance_config); From 600d146b19f9ab352fb391ae08079d676a8aa7ee Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 19:05:20 +0300 Subject: [PATCH 14/14] tests: remove tests for page healing via pg_ptrack_get_block --- tests/backup.py | 133 ++++++---------------------------------- tests/delta.py | 157 ------------------------------------------------ 2 files changed, 20 insertions(+), 270 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 176aba61..694c9a44 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -147,10 +147,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): "without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "ERROR: Valid backup on current timeline 1 is not found. " - "Create new FULL backup before an incremental one.", - e.message, + self.assertTrue( + "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -165,10 +164,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): "without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "ERROR: Valid backup on current timeline 1 is not found. " - "Create new FULL backup before an incremental one.", - e.message, + self.assertTrue( + "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -176,6 +174,10 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], + "ERROR") + # Clean after yourself self.del_test_dir(module_name, fname) @@ -315,10 +317,8 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack_1(self): + def test_page_detect_corruption(self): """make node, corrupt some page, check that backup failed""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( @@ -380,98 +380,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack_2(self): - """make node, corrupt some page, check that backup failed""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").rstrip() - node.stop() - - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", '--log-level-console=LOG']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of page " - "corruption in PostgreSQL instance.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - if self.remote: - self.assertTrue( - "WARNING: File" in e.message and - "try to fetch via shared buffer" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - else: - self.assertTrue( - "LOG: File" in e.message and - "blknum" in e.message and - "have wrong checksum" in e.message and - "try to fetch via shared buffer" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', - "Backup Status should be ERROR") - - # Clean after yourself - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_backup_detect_corruption(self): @@ -495,6 +403,10 @@ class BackupTest(ProbackupTest, unittest.TestCase): "postgres", "create extension ptrack") + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + node.safe_psql( "postgres", "create table t_heap as select 1 as id, md5(i::text) as text, " @@ -529,10 +441,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): node.slow_start() - # self.backup_node( - # backup_dir, 'node', node, - # backup_type="full", options=["-j", "4", "--stream"]) - try: self.backup_node( backup_dir, 'node', node, @@ -608,12 +516,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'WARNING: page verification failed, ' - 'calculated checksum' in e.message and - 'ERROR: query failed: ERROR: ' - 'invalid page in block 1 of relation' in e.message and - 'ERROR: Data files transferring failed' in e.message, + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/delta.py b/tests/delta.py index 4447b9a8..fdbaf127 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1050,163 +1050,6 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_delta_corruption_heal_via_ptrack(self): - """make node, corrupt some page, check that backup failed""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").rstrip() - - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", - options=["-j", "4", "--stream", '--log-level-file=verbose']) - - # open log file and check - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertIn('block 1, try to fetch via shared buffer', log_content) - self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) - f.close - - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', - "Backup Status should be OK") - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack(self): - """make node, corrupt some page, check that backup failed""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").rstrip() - node.stop() - - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=["-j", "4", "--stream", "--log-level-console=LOG"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of page " - "corruption in PostgreSQL instance.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - if self.remote: - self.assertTrue( - "LOG: File" in e.message and - "try to fetch via shared buffer" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - else: - self.assertTrue( - "WARNING: File" in e.message and - "blknum" in e.message and - "have wrong checksum" in e.message and - "try to fetch via shared buffer" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', - "Backup Status should be ERROR") - - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_delta_nullified_heap_page_backup(self): """ make node, take full backup, nullify some heap block,