From 4155171b1923bce38e1c83cb6d21a99ce3f95de3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 30 Nov 2018 09:19:32 +0300 Subject: [PATCH 01/59] update README.md about PG11 support --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 53945eb8..5c2267d0 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,6 @@ The utility is compatible with: * PostgreSQL 9.5, 9.6, 10, 11; -`PTRACK` backup support provided via following options: -* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) -* Postgres Pro Standard 9.5, 9.6, 10, 11 -* Postgres Pro Enterprise 9.5, 9.6, 10 - As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Choosing between full and page-level incremental backups to speed up backup and recovery * Implementing a single backup strategy for multi-server PostgreSQL clusters @@ -34,6 +29,11 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp * `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. * `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). +`PTRACK` backup support provided via following options: +* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) +* Postgres Pro Standard 9.5, 9.6, 10, 11 +* Postgres Pro Enterprise 9.5, 9.6, 10 + ## Limitations `pg_probackup` currently has the following limitations: @@ -48,27 +48,27 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp #DEB Ubuntu|Debian Packages echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update -apt-get install pg-probackup-{10,9.6,9.5} +apt-get install pg-probackup-{11,10,9.6,9.5} #DEB-SRC Packages echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list -apt-get source pg-probackup-{10,9.6,9.5} +apt-get source pg-probackup-{11,10,9.6,9.5} #RPM Centos Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5} #RPM RHEL Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5} #RPM Oracle Linux Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5} #SRPM Packages -yumdownloader --source pg_probackup-{10,9.6,9.5} +yumdownloader --source pg_probackup-{11,10,9.6,9.5} ``` To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: From a039fd14aa641d8427cabd9dc0caae75dd21da19 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 3 Dec 2018 13:38:45 +0300 Subject: [PATCH 02/59] add warning if waiting for START LSN during ARCHIVE backup is exceeding 30 seconds --- src/backup.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/backup.c b/src/backup.c index d9238929..f308aa89 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1665,6 +1665,11 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); } + if (!stream_wal && is_start_lsn && try_count == 30) + elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " + "If continius archiving is not set up, use '--stream' option to make autonomous backup. " + "Otherwise check that continius archiving works correctly."); + if (timeout > 0 && try_count > timeout) { if (file_exists) From 529173287f5937a22aaa8dc8aab628c708186242 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 3 Dec 2018 13:41:39 +0300 Subject: [PATCH 03/59] fix tabulation offset --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index f308aa89..7046d1d0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1667,8 +1667,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) if (!stream_wal && is_start_lsn && try_count == 30) elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " - "If continius archiving is not set up, use '--stream' option to make autonomous backup. " - "Otherwise check that continius archiving works correctly."); + "If continius archiving is not set up, use '--stream' option to make autonomous backup. " + "Otherwise check that continius archiving works correctly."); if (timeout > 0 && try_count > timeout) { From 4ac0fab61b21d80c260078b04453f3b6a1b9e16d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 6 Dec 2018 16:03:29 +0300 Subject: [PATCH 04/59] Show progress messages with INFO verbose level --- src/delete.c | 10 ++++++---- src/merge.c | 2 +- src/restore.c | 2 +- src/validate.c | 9 +++++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/delete.c b/src/delete.c index e7c9ac24..8e9b965f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -256,6 +256,7 @@ delete_backup_files(pgBackup *backup) char path[MAXPGPATH]; char timestamp[100]; parray *files; + size_t num_files; /* * If the backup was deleted already, there is nothing to do. @@ -286,13 +287,14 @@ delete_backup_files(pgBackup *backup) /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); - for (i = 0; i < parray_num(files); i++) + num_files = parray_num(files); + for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(files, i); - /* print progress */ - elog(VERBOSE, "Delete file(%zd/%lu) \"%s\"", i + 1, - (unsigned long) parray_num(files), file->path); + if (progress) + elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"", + i + 1, num_files, file->path); if (remove(file->path)) { diff --git a/src/merge.c b/src/merge.c index b01a90d0..2a24b39d 100644 --- a/src/merge.c +++ b/src/merge.c @@ -385,7 +385,7 @@ merge_files(void *arg) elog(ERROR, "Interrupted during merging backups"); if (progress) - elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, num_files, file->path); /* diff --git a/src/restore.c b/src/restore.c index 6024f8bc..8770c358 100644 --- a/src/restore.c +++ b/src/restore.c @@ -582,7 +582,7 @@ restore_files(void *arg) rel_path = GetRelativePath(file->path,from_root); if (progress) - elog(LOG, "Progress: (%d/%lu). Process file %s ", + elog(INFO, "Progress: (%d/%lu). Process file %s ", i + 1, (unsigned long) parray_num(arguments->files), rel_path); /* diff --git a/src/validate.c b/src/validate.c index 1f758d0f..edb386ef 100644 --- a/src/validate.c +++ b/src/validate.c @@ -156,9 +156,10 @@ pgBackupValidateFiles(void *arg) { int i; validate_files_arg *arguments = (validate_files_arg *)arg; + int num_files = parray_num(arguments->files); pg_crc32 crc; - for (i = 0; i < parray_num(arguments->files); i++) + for (i = 0; i < num_files; i++) { struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); @@ -186,9 +187,9 @@ pgBackupValidateFiles(void *arg) if (file->is_cfs) continue; - /* print progress */ - elog(VERBOSE, "Validate files: (%d/%lu) %s", - i + 1, (unsigned long) parray_num(arguments->files), file->path); + if (progress) + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); if (stat(file->path, &st) == -1) { From e753643b8d65ad2656bca485f962de348f279433 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 6 Dec 2018 19:00:08 +0300 Subject: [PATCH 05/59] Add error message in case of invalid compression algorithm --- src/data.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/data.c b/src/data.c index 19bb742b..de649103 100644 --- a/src/data.c +++ b/src/data.c @@ -98,6 +98,8 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, { case NONE_COMPRESS: case NOT_DEFINED_COMPRESS: + if (errormsg) + *errormsg = "Invalid compression algorithm"; return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: From 8b1b53bc3bc17d1c4894db1fa618e46e125f5980 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 6 Dec 2018 19:07:56 +0300 Subject: [PATCH 06/59] tests: test_merge_compressed_and_uncompressed_backups added --- tests/merge.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 5f7ae7da..5a314dcf 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -163,6 +163,87 @@ class MergeTest(ProbackupTest, unittest.TestCase): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[1] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "DELTA") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do not compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_merge_tablespaces(self): """ From 34c05740b05dd1ee89745d9b4678e1220ed6b812 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 6 Dec 2018 19:41:10 +0300 Subject: [PATCH 07/59] Restore to_backup sizes after copying metadata from from_backup --- src/merge.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/merge.c b/src/merge.c index 2a24b39d..6d6d978e 100644 --- a/src/merge.c +++ b/src/merge.c @@ -168,6 +168,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) merge_files_arg *threads_args = NULL; int i; bool merge_isok = true; + int64 to_data_bytes, + to_wal_bytes; elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); @@ -282,26 +284,28 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ to_backup->status = BACKUP_STATUS_OK; /* Compute summary of size of regular files in the backup */ - to_backup->data_bytes = 0; + to_data_bytes = 0; for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); if (S_ISDIR(file->mode)) - to_backup->data_bytes += 4096; + to_data_bytes += 4096; /* Count the amount of the data actually copied */ else if (S_ISREG(file->mode)) - to_backup->data_bytes += file->write_size; + to_data_bytes += file->write_size; } /* compute size of wal files of this backup stored in the archive */ if (!to_backup->stream) - to_backup->wal_bytes = instance_config.xlog_seg_size * + to_wal_bytes = instance_config.xlog_seg_size * (to_backup->stop_lsn / instance_config.xlog_seg_size - to_backup->start_lsn / instance_config.xlog_seg_size + 1); else - to_backup->wal_bytes = BYTES_INVALID; + to_wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path); + to_backup->data_bytes = to_data_bytes; + to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); delete_source_backup: @@ -314,6 +318,7 @@ delete_source_backup: /* * Delete files which are not in from_backup file list. */ + parray_qsort(files, pgFileComparePathDesc); for (i = 0; i < parray_num(to_files); i++) { pgFile *file = (pgFile *) parray_get(to_files, i); @@ -341,6 +346,9 @@ delete_source_backup: to_backup->backup_mode = BACKUP_MODE_FULL; to_backup->status = BACKUP_STATUS_OK; to_backup->parent_backup = INVALID_BACKUP_ID; + /* Restore sizes */ + to_backup->data_bytes = to_data_bytes; + to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); /* Cleanup */ From 59ab21e5c2be7c311b8474a714ff6880c398e790 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Dec 2018 09:43:38 +0300 Subject: [PATCH 08/59] tests: test_merge_compressed_backups_1 added --- tests/merge.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 5a314dcf..cbe84d5c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -163,6 +163,83 @@ class MergeTest(ProbackupTest, unittest.TestCase): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_backups_1(self): + """ + Test MERGE command with compressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups(self): """ Test MERGE command with compressed and uncompressed backups @@ -203,19 +280,14 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, "node", node, backup_type="delta", options=['--compress-algorithm=zlib', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[1] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "DELTA") # Change data pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) pgbench.wait() - # Do not compressed PAGE backup + # Do uncompressed PAGE backup self.backup_node( - backup_dir, "node", node, backup_type="page", - options=['--compress-algorithm=zlib']) + backup_dir, "node", node, backup_type="page") pgdata = self.pgdata_content(node.data_dir) From 2837399e6bdb9ccde4a28e68aa3e9757af4dabb5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 7 Dec 2018 17:24:21 +0300 Subject: [PATCH 09/59] Fix MERGE of compressed and uncompressed backups --- src/merge.c | 116 ++++++++++++++++++++++--------------------------- tests/merge.py | 2 - 2 files changed, 51 insertions(+), 67 deletions(-) diff --git a/src/merge.c b/src/merge.c index 6d6d978e..e0669586 100644 --- a/src/merge.c +++ b/src/merge.c @@ -131,7 +131,6 @@ do_merge(time_t backup_id) { pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); - full_backup = (pgBackup *) parray_get(backups, i); merge_backups(full_backup, from_backup); } @@ -168,8 +167,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) merge_files_arg *threads_args = NULL; int i; bool merge_isok = true; - int64 to_data_bytes, - to_wal_bytes; elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); @@ -283,29 +280,37 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * Update to_backup metadata. */ to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + to_backup->start_lsn = from_backup->start_lsn; + to_backup->stop_lsn = from_backup->stop_lsn; + to_backup->recovery_time = from_backup->recovery_time; + to_backup->recovery_xid = from_backup->recovery_xid; + /* + * If one of the backups isn't "stream" backup then the target backup become + * non-stream backup too. + */ + to_backup->stream = to_backup->stream && from_backup->stream; /* Compute summary of size of regular files in the backup */ - to_data_bytes = 0; + to_backup->data_bytes = 0; for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); if (S_ISDIR(file->mode)) - to_data_bytes += 4096; + to_backup->data_bytes += 4096; /* Count the amount of the data actually copied */ else if (S_ISREG(file->mode)) - to_data_bytes += file->write_size; + to_backup->data_bytes += file->write_size; } /* compute size of wal files of this backup stored in the archive */ if (!to_backup->stream) - to_wal_bytes = instance_config.xlog_seg_size * + to_backup->wal_bytes = instance_config.xlog_seg_size * (to_backup->stop_lsn / instance_config.xlog_seg_size - to_backup->start_lsn / instance_config.xlog_seg_size + 1); else - to_wal_bytes = BYTES_INVALID; + to_backup->wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path); - to_backup->data_bytes = to_data_bytes; - to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); delete_source_backup: @@ -330,27 +335,6 @@ delete_source_backup: } } - /* - * Rename FULL backup directory. - */ - elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id); - if (rename(to_backup_path, from_backup_path) == -1) - elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", - to_backup_path, from_backup_path, strerror(errno)); - - /* - * Merging finished, now we can safely update ID of the destination backup. - */ - pgBackupCopy(to_backup, from_backup); - /* Correct metadata */ - to_backup->backup_mode = BACKUP_MODE_FULL; - to_backup->status = BACKUP_STATUS_OK; - to_backup->parent_backup = INVALID_BACKUP_ID; - /* Restore sizes */ - to_backup->data_bytes = to_data_bytes; - to_backup->wal_bytes = to_wal_bytes; - write_backup(to_backup); - /* Cleanup */ if (threads) { @@ -384,6 +368,8 @@ merge_files(void *arg) for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(argument->files, i); + pgFile *to_file; + pgFile **res_file; if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -392,17 +378,24 @@ merge_files(void *arg) if (interrupted) elog(ERROR, "Interrupted during merging backups"); + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + if (progress) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, num_files, file->path); + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + to_file = (res_file) ? *res_file : NULL; + /* * Skip files which haven't changed since previous backup. But in case * of DELTA backup we should consider n_blocks to truncate the target * backup. */ - if (file->write_size == BYTES_INVALID && - file->n_blocks == -1) + if (file->write_size == BYTES_INVALID && file->n_blocks == -1) { elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", file->path); @@ -411,27 +404,16 @@ merge_files(void *arg) * If the file wasn't changed in PAGE backup, retreive its * write_size from previous FULL backup. */ - if (S_ISREG(file->mode)) + if (to_file) { - pgFile **res_file; - - res_file = parray_bsearch(argument->to_files, file, - pgFileComparePathDesc); - if (res_file && *res_file) - { - file->compress_alg = (*res_file)->compress_alg; - file->write_size = (*res_file)->write_size; - file->crc = (*res_file)->crc; - } + file->compress_alg = to_file->compress_alg; + file->write_size = to_file->write_size; + file->crc = to_file->crc; } continue; } - /* Directories were created before */ - if (S_ISDIR(file->mode)) - continue; - /* * Move the file. We need to decompress it and compress again if * necessary. @@ -447,7 +429,7 @@ merge_files(void *arg) file->path + to_root_len + 1); /* - * We need more complicate algorithm if target file exists and it is + * We need more complicate algorithm if target file should be * compressed. */ if (to_backup->compress_alg == PGLZ_COMPRESS || @@ -462,33 +444,36 @@ merge_files(void *arg) /* * Merge files: - * - decompress first file - * - decompress second file and merge with first decompressed file + * - if target file exists restore and decompress it to the temp + * path + * - decompress source file if necessary and merge it with the + * target decompressed file * - compress result file */ - elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", - tmp_file_path); - - prev_path = file->path; /* - * We need to decompress target file only if it exists. + * We need to decompress target file if it exists. */ - if (fileExists(to_path_tmp)) + if (to_file) { + elog(VERBOSE, "Merge target and source files into the temporary path \"%s\"", + tmp_file_path); + /* * file->path points to the file in from_root directory. But we * need the file in directory to_root. */ - file->path = to_path_tmp; - - /* Decompress first/target file */ - restore_data_file(tmp_file_path, file, false, false, + prev_path = to_file->path; + to_file->path = to_path_tmp; + /* Decompress target file into temporary one */ + restore_data_file(tmp_file_path, to_file, false, false, parse_program_version(to_backup->program_version)); - - file->path = prev_path; + to_file->path = prev_path; } - /* Merge second/source file with first/target file */ + else + elog(VERBOSE, "Restore source file into the temporary path \"%s\"", + tmp_file_path); + /* Merge source file with target file */ restore_data_file(tmp_file_path, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, false, @@ -497,7 +482,8 @@ merge_files(void *arg) elog(VERBOSE, "Compress file and save it to the directory \"%s\"", argument->to_root); - /* Again we need change path */ + /* Again we need to change path */ + prev_path = file->path; file->path = tmp_file_path; /* backup_data_file() requires file size to calculate nblocks */ file->size = pgFileSize(file->path); diff --git a/tests/merge.py b/tests/merge.py index cbe84d5c..c3b3169a 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -875,8 +875,6 @@ class MergeTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Backup {0} data files are corrupted".format( - backup_id) in e.message and "ERROR: Merging of backup {0} failed".format( backup_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( From b455420a18b1470726c3b44714cbc74800d40df2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Dec 2018 23:39:39 +0300 Subject: [PATCH 10/59] tests: more merge tests added --- tests/merge.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index c3b3169a..e4209441 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -316,6 +316,159 @@ class MergeTest(ProbackupTest, unittest.TestCase): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups_1(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_and_uncompressed_backups_2(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do uncompressed FULL backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") def test_merge_tablespaces(self): """ From c1903db22b0812f3b327a8e3a0cba3299413d7d2 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 16:19:51 +0300 Subject: [PATCH 11/59] Fix format warnings --- src/archive.c | 4 ++-- src/data.c | 12 ++++++------ src/restore.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/archive.c b/src/archive.c index 3333f096..774d12c4 100644 --- a/src/archive.c +++ b/src/archive.c @@ -27,7 +27,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) char backup_wal_file_path[MAXPGPATH]; char absolute_wal_file_path[MAXPGPATH]; char current_dir[MAXPGPATH]; - int64 system_id; + uint64 system_id; bool is_compress = false; if (wal_file_name == NULL && wal_file_path == NULL) @@ -50,7 +50,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) if(system_id != instance_config.system_identifier) elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." - "Instance '%s' should have SYSTEM_ID = %ld instead of %ld", + "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, wal_file_name, instance_name, instance_config.system_identifier, system_id); diff --git a/src/data.c b/src/data.c index de649103..0d0c5e5c 100644 --- a/src/data.c +++ b/src/data.c @@ -232,8 +232,8 @@ read_page_from_file(pgFile *file, BlockNumber blknum, return 0; } else - elog(WARNING, "File: %s, block %u, expected block size %d," - "but read %lu, try again", + elog(WARNING, "File: %s, block %u, expected block size %u," + "but read %zu, try again", file->path, blknum, BLCKSZ, read_len); } @@ -382,7 +382,7 @@ prepare_page(backup_files_arg *arguments, else if (page_size != BLCKSZ) { free(ptrack_page); - elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", + elog(ERROR, "File: %s, block %u, expected block size %d, but read %zu", file->path, absolute_blknum, BLCKSZ, page_size); } else @@ -574,7 +574,7 @@ backup_data_file(backup_files_arg* arguments, if (file->size % BLCKSZ != 0) { fclose(in); - elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size); + elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size); } /* @@ -789,7 +789,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); /* @@ -1620,7 +1620,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); diff --git a/src/restore.c b/src/restore.c index 8770c358..e3591e06 100644 --- a/src/restore.c +++ b/src/restore.c @@ -480,7 +480,7 @@ restore_backup(pgBackup *backup) /* By default there are some error */ threads_args[i].ret = 1; - elog(LOG, "Start thread for num:%li", parray_num(files)); + elog(LOG, "Start thread for num:%zu", parray_num(files)); pthread_create(&threads[i], NULL, restore_files, arg); } From dd18928e1984927edafe175a2a1bb30f9f64d66e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 18:02:22 +0300 Subject: [PATCH 12/59] Make compiler happy --- src/utils/logger.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index fa1c0039..a4f2d692 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -232,18 +232,23 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) * Write to stderr if the message was not written to log file. * Write to stderr if the message level is greater than WARNING anyway. */ - if (write_to_stderr) + if (write_to_stderr && write_to_file) { write_elevel(stderr, elevel); - if (write_to_file) - vfprintf(stderr, fmt, std_args); - else - vfprintf(stderr, fmt, args); + + vfprintf(stderr, fmt, std_args); fputc('\n', stderr); fflush(stderr); - if (write_to_file) - va_end(std_args); + va_end(std_args); + } + else if (write_to_stderr) + { + write_elevel(stderr, elevel); + + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); } exit_if_necessary(elevel); From 9ac30ada5443e97de9152de7bab5c3b9752b2abb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 10 Dec 2018 18:49:39 +0300 Subject: [PATCH 13/59] bugfix: correctly log cmdline to logfile --- src/pg_probackup.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 17676876..04b5d486 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -331,15 +331,6 @@ main(int argc, char *argv[]) if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) elog(ERROR, "-B, --backup-path must be a path to directory"); - /* command was initialized for a few commands */ - if (command) - { - elog_file(INFO, "command: %s", command); - - pfree(command); - command = NULL; - } - /* Option --instance is required for all commands except init and show */ if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && backup_subcmd != VALIDATE_CMD) @@ -390,6 +381,15 @@ main(int argc, char *argv[]) /* Initialize logger */ init_logger(backup_path, &instance_config.logger); + /* command was initialized for a few commands */ + if (command) + { + elog_file(INFO, "command: %s", command); + + pfree(command); + command = NULL; + } + /* * We have read pgdata path from command line or from configuration file. * Ensure that pgdata is an absolute path. From eead6b6307b464b67e567516609baac7fce7560c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 19:00:37 +0300 Subject: [PATCH 14/59] Free unnecessary file object --- src/dir.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dir.c b/src/dir.c index 710598b7..ae8cb20b 100644 --- a/src/dir.c +++ b/src/dir.c @@ -443,6 +443,9 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(files, file); dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); + + if (!add_root) + pgFileFree(file); } /* From 385f979fe7daf91001eea4f4c123807bb6f576ee Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 11 Dec 2018 12:27:36 +0300 Subject: [PATCH 15/59] Make compiler happy again --- src/utils/logger.c | 98 ++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index a4f2d692..ba054a62 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -40,10 +40,10 @@ typedef enum void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); -static void elog_internal(int elevel, bool file_only, const char *fmt, va_list args) - pg_attribute_printf(3, 0); +static void elog_internal(int elevel, bool file_only, const char *message); static void elog_stderr(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); +static char *get_log_message(const char *fmt, va_list args) pg_attribute_printf(1, 0); /* Functions to work with log files */ static void open_logfile(FILE **file, const char *filename_format); @@ -74,7 +74,7 @@ init_logger(const char *root_path, LoggerConfig *config) /* Set log path */ if (config->log_directory == NULL) { - config->log_directory = palloc(MAXPGPATH); + config->log_directory = pgut_malloc(MAXPGPATH); join_path_components(config->log_directory, root_path, LOG_DIRECTORY_DEFAULT); } @@ -148,13 +148,11 @@ exit_if_necessary(int elevel) * Actual implementation for elog() and pg_log(). */ static void -elog_internal(int elevel, bool file_only, const char *fmt, va_list args) +elog_internal(int elevel, bool file_only, const char *message) { bool write_to_file, write_to_error_log, write_to_stderr; - va_list error_args, - std_args; time_t log_time = (time_t) time(NULL); char strfbuf[128]; @@ -165,22 +163,8 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) write_to_stderr = elevel >= logger_config.log_level_console && !file_only; pthread_lock(&log_file_mutex); -#ifdef WIN32 - std_args = NULL; - error_args = NULL; -#endif loggin_in_progress = true; - /* We need copy args only if we need write to error log file */ - if (write_to_error_log) - va_copy(error_args, args); - /* - * We need copy args only if we need write to stderr. But do not copy args - * if we need to log only to stderr. - */ - if (write_to_stderr && write_to_file) - va_copy(std_args, args); - if (write_to_file || write_to_error_log) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", localtime(&log_time)); @@ -203,8 +187,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(log_file, "%s: ", strfbuf); write_elevel(log_file, elevel); - vfprintf(log_file, fmt, args); - fputc('\n', log_file); + fprintf(log_file, "%s\n", message); fflush(log_file); } @@ -221,33 +204,19 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(error_log_file, "%s: ", strfbuf); write_elevel(error_log_file, elevel); - vfprintf(error_log_file, fmt, error_args); - fputc('\n', error_log_file); + fprintf(error_log_file, "%s\n", message); fflush(error_log_file); - - va_end(error_args); } /* * Write to stderr if the message was not written to log file. * Write to stderr if the message level is greater than WARNING anyway. */ - if (write_to_stderr && write_to_file) + if (write_to_stderr) { write_elevel(stderr, elevel); - vfprintf(stderr, fmt, std_args); - fputc('\n', stderr); - fflush(stderr); - - va_end(std_args); - } - else if (write_to_stderr) - { - write_elevel(stderr, elevel); - - vfprintf(stderr, fmt, args); - fputc('\n', stderr); + fprintf(stderr, "%s\n", message); fflush(stderr); } @@ -285,12 +254,44 @@ elog_stderr(int elevel, const char *fmt, ...) exit_if_necessary(elevel); } +/* + * Formats text data under the control of fmt and returns it in an allocated + * buffer. + */ +static char * +get_log_message(const char *fmt, va_list args) +{ + size_t len = 256; /* initial assumption about buffer size */ + + for (;;) + { + char *result; + size_t newlen; + va_list copy_args; + + result = (char *) pgut_malloc(len); + + /* Try to format the data */ + va_copy(copy_args, args); + newlen = pvsnprintf(result, len, fmt, copy_args); + va_end(copy_args); + + if (newlen < len) + return result; /* success */ + + /* Release buffer and loop around to try again with larger len. */ + pfree(result); + len = newlen; + } +} + /* * Logs to stderr or to log file and exit if ERROR. */ void elog(int elevel, const char *fmt, ...) { + char *message; va_list args; /* @@ -302,8 +303,11 @@ elog(int elevel, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, false, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, false, message); + pfree(message); } /* @@ -312,6 +316,7 @@ elog(int elevel, const char *fmt, ...) void elog_file(int elevel, const char *fmt, ...) { + char *message; va_list args; /* @@ -322,8 +327,11 @@ elog_file(int elevel, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, true, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, true, message); + pfree(message); } /* @@ -332,6 +340,7 @@ elog_file(int elevel, const char *fmt, ...) void pg_log(eLogType type, const char *fmt, ...) { + char *message; va_list args; int elevel = INFO; @@ -364,8 +373,11 @@ pg_log(eLogType type, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, false, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, false, message); + pfree(message); } /* @@ -450,7 +462,7 @@ logfile_getname(const char *format, time_t timestamp) logger_config.log_directory[0] == '\0') elog_stderr(ERROR, "logging path is not set"); - filename = (char *) palloc(MAXPGPATH); + filename = (char *) pgut_malloc(MAXPGPATH); snprintf(filename, MAXPGPATH, "%s/", logger_config.log_directory); From eed59874069c05026ae76c04355ead8f9795d9d9 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 11 Dec 2018 12:47:21 +0300 Subject: [PATCH 16/59] Remove BOM marker from configuration.c --- src/utils/configuration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 9b88d159..fe50c494 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1,4 +1,4 @@ -/*------------------------------------------------------------------------- +/*------------------------------------------------------------------------- * * configuration.c: - function implementations to work with pg_probackup * configurations. From 30cf45816d27b2bb877bc54689f71f09483dc004 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 12 Dec 2018 00:29:20 +0300 Subject: [PATCH 17/59] check backup version before backup validation, throw an error if backup version > binary version --- src/validate.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/validate.c b/src/validate.c index edb386ef..14d347ca 100644 --- a/src/validate.c +++ b/src/validate.c @@ -52,6 +52,14 @@ pgBackupValidate(pgBackup *backup) validate_files_arg *threads_args; int i; + /* Check backup version */ + if (backup->program_version && + parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) + elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version); + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE && From 23d1a26e024c3fb66f4d5c514abd80ed53558cae Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 12 Dec 2018 15:10:25 +0300 Subject: [PATCH 18/59] Fix MERGE of uncompressed with compressed backups --- src/catalog.c | 14 -------------- src/data.c | 23 +++++++++++++++++------ src/dir.c | 2 +- src/merge.c | 23 ++++++++++++++++++++++- src/pg_probackup.h | 1 - 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 77ac0f34..b30fcc66 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -820,20 +820,6 @@ pgBackupInit(pgBackup *backup) backup->server_version[0] = '\0'; } -/* - * Copy backup metadata from **src** into **dst**. - */ -void -pgBackupCopy(pgBackup *dst, pgBackup *src) -{ - pfree(dst->primary_conninfo); - - memcpy(dst, src, sizeof(pgBackup)); - - if (src->primary_conninfo) - dst->primary_conninfo = pstrdup(src->primary_conninfo); -} - /* free pgBackup object */ void pgBackupFree(void *backup) diff --git a/src/data.c b/src/data.c index 0d0c5e5c..ccbe9ca8 100644 --- a/src/data.c +++ b/src/data.c @@ -829,6 +829,8 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, if (write_header) { + /* We uncompressed the page, so its size is BLCKSZ */ + header.compressed_size = BLCKSZ; if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) elog(ERROR, "cannot write header of block %u of \"%s\": %s", blknum, file->path, strerror(errno)); @@ -1592,19 +1594,23 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (read_len == 0 && feof(in)) break; /* EOF found */ else if (read_len != 0 && feof(in)) - elog(ERROR, + elog(WARNING, "odd size page found at block %u of \"%s\"", blknum, file->path); else - elog(ERROR, "cannot read header of block %u of \"%s\": %s", + elog(WARNING, "cannot read header of block %u of \"%s\": %s", blknum, file->path, strerror(errno_tmp)); + return false; } COMP_FILE_CRC32(use_crc32c, crc, &header, read_len); if (header.block < blknum) - elog(ERROR, "backup is broken at file->path %s block %u", + { + elog(WARNING, "backup is broken at file->path %s block %u", file->path, blknum); + return false; + } blknum = header.block; @@ -1620,8 +1626,11 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", + { + elog(WARNING, "cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); + return false; + } COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); @@ -1648,11 +1657,13 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, is_valid = false; continue; } - elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + elog(WARNING, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", file->path, uncompressed_size); + return false; } + if (validate_one_page(page.data, file, blknum, - stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID) + stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID) is_valid = false; } else diff --git a/src/dir.c b/src/dir.c index ae8cb20b..386f49bc 100644 --- a/src/dir.c +++ b/src/dir.c @@ -211,7 +211,7 @@ pgFileInit(const char *path) strcpy(file->path, path); /* enough buffer size guaranteed */ /* Get file name from the path */ - file_name = strrchr(file->path, '/'); + file_name = last_dir_separator(file->path); if (file_name == NULL) file->name = file->path; else diff --git a/src/merge.c b/src/merge.c index e0669586..b936469b 100644 --- a/src/merge.c +++ b/src/merge.c @@ -91,7 +91,8 @@ do_merge(time_t backup_id) } else { - Assert(dest_backup); + if (dest_backup == NULL) + elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); if (backup->start_time != prev_parent) continue; @@ -335,6 +336,20 @@ delete_source_backup: } } + /* + * Rename FULL backup directory. + */ + elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id); + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Merging finished, now we can safely update ID of the destination backup. + */ + to_backup->start_time = from_backup->start_time; + write_backup(to_backup); + /* Cleanup */ if (threads) { @@ -526,6 +541,12 @@ merge_files(void *arg) else copy_file(argument->from_root, argument->to_root, file); + /* + * We need to save compression algorithm type of the target backup to be + * able to restore in the future. + */ + file->compress_alg = to_backup->compress_alg; + if (file->write_size != BYTES_INVALID) elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", file->path, file->write_size); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0dc649a2..546c1016 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -476,7 +476,6 @@ extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); extern void pgBackupInit(pgBackup *backup); -extern void pgBackupCopy(pgBackup *dst, pgBackup *src); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); extern int pgBackupCompareIdDesc(const void *f1, const void *f2); From 0b05cf0343a688178fafa33ad916c81875287234 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 12 Dec 2018 17:27:31 +0300 Subject: [PATCH 19/59] Improve comment within on_interrupt() --- src/utils/pgut.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index ccf832b9..e08b53ea 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -703,7 +703,10 @@ on_interrupt(void) /* Set interruped flag */ interrupted = true; - /* User promts password, call on_cleanup() byhand */ + /* + * User promts password, call on_cleanup() byhand. If we don't do that we + * will stuck forever until an user enters a password. + */ if (in_password) { on_cleanup(); From 5ad509a40efec5af6ce22c5cd8c08b2dee94147b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 12 Dec 2018 17:52:01 +0300 Subject: [PATCH 20/59] Fix comment introduced in 0b05cf0343 --- src/utils/pgut.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index e08b53ea..a7e12e91 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -704,8 +704,8 @@ on_interrupt(void) interrupted = true; /* - * User promts password, call on_cleanup() byhand. If we don't do that we - * will stuck forever until an user enters a password. + * User promts password, call on_cleanup() byhand. Unless we do that we will + * get stuck forever until a user enters a password. */ if (in_password) { From 9484b6de27c3768e7105a87f5c94e989611767fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 12 Dec 2018 20:32:42 +0300 Subject: [PATCH 21/59] tests: test_page_create_db added --- tests/page.py | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/tests/page.py b/tests/page.py index f0647082..d0ebbb9b 100644 --- a/tests/page.py +++ b/tests/page.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import QueryException from datetime import datetime, timedelta import subprocess import gzip @@ -1030,3 +1031,120 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_create_db(self): + """ + Make node, take full backup, create database db1, take page backup, + restore database and check it presense + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + self.backup_node( + backup_dir, 'node', node) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1000) i") + + # PAGE BACKUP + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + node_restored.safe_psql('db1', 'select 1') + node_restored.cleanup() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND PTRACK BACKUP + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) From eef5769ffd3da14e9d4faa649b0b162dbd606af6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 13 Dec 2018 10:20:49 +0300 Subject: [PATCH 22/59] tests: pgdata_compare now detect missing or redundant directories --- tests/helpers/ptrack_helpers.py | 48 ++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d81d7288..0bc27abf 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1047,7 +1047,7 @@ class ProbackupTest(object): except: pass - def pgdata_content(self, directory, ignore_ptrack=True): + def pgdata_content(self, pgdata, ignore_ptrack=True): """ return dict with directory content. " " TAKE IT AFTER CHECKPOINT or BACKUP""" dirs_to_ignore = [ @@ -1064,9 +1064,10 @@ class ProbackupTest(object): # '_ptrack' # ) directory_dict = {} - directory_dict['pgdata'] = directory + directory_dict['pgdata'] = pgdata directory_dict['files'] = {} - for root, dirs, files in os.walk(directory, followlinks=True): + directory_dict['dirs'] = [] + for root, dirs, files in os.walk(pgdata, followlinks=True): dirs[:] = [d for d in dirs if d not in dirs_to_ignore] for file in files: if ( @@ -1076,7 +1077,7 @@ class ProbackupTest(object): continue file_fullpath = os.path.join(root, file) - file_relpath = os.path.relpath(file_fullpath, directory) + file_relpath = os.path.relpath(file_fullpath, pgdata) directory_dict['files'][file_relpath] = {'is_datafile': False} directory_dict['files'][file_relpath]['md5'] = hashlib.md5( open(file_fullpath, 'rb').read()).hexdigest() @@ -1089,12 +1090,51 @@ class ProbackupTest(object): file_fullpath, size_in_pages ) + for root, dirs, files in os.walk(pgdata, topdown=False, followlinks=True): + for directory in dirs: + directory_path = os.path.join(root, directory) + directory_relpath = os.path.relpath(directory_path, pgdata) + + found = False + for d in dirs_to_ignore: + if d in directory_relpath: + found = True + break + + # check if directory already here as part of larger directory + if not found: + for d in directory_dict['dirs']: + # print("OLD dir {0}".format(d)) + if directory_relpath in d: + found = True + break + + if not found: + directory_dict['dirs'].append(directory_relpath) + return directory_dict def compare_pgdata(self, original_pgdata, restored_pgdata): """ return dict with directory content. DO IT BEFORE RECOVERY""" fail = False error_message = 'Restored PGDATA is not equal to original!\n' + + # Compare directories + for directory in restored_pgdata['dirs']: + if directory not in original_pgdata['dirs']: + fail = True + error_message += '\nDirectory is not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory)) + + for directory in original_pgdata['dirs']: + if directory not in restored_pgdata['dirs']: + fail = True + error_message += '\nDirectory dissappeared' + error_message += ' in restored PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory)) + + for file in restored_pgdata['files']: # File is present in RESTORED PGDATA # but not present in ORIGINAL From a5cef9890829c90b4df15efee6b5e5cda6c9c782 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 13 Dec 2018 11:45:04 +0300 Subject: [PATCH 23/59] tests: test_merge_tablespaces and test_merge_tablespaces_1 added --- tests/helpers/ptrack_helpers.py | 2 +- tests/merge.py | 154 ++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 0bc27abf..8ea90c24 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1123,7 +1123,7 @@ class ProbackupTest(object): for directory in restored_pgdata['dirs']: if directory not in original_pgdata['dirs']: fail = True - error_message += '\nDirectory is not present' + error_message += '\nDirectory was not present' error_message += ' in original PGDATA: {0}\n'.format( os.path.join(restored_pgdata['pgdata'], directory)) diff --git a/tests/merge.py b/tests/merge.py index e4209441..8ce1795e 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -3,6 +3,7 @@ import unittest import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import shutil module_name = "merge" @@ -471,9 +472,162 @@ class MergeTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_merge_tablespaces(self): + """ + Create tablespace with table + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # Create new tablespace + self.create_tblspace_in_node(node, 'somedata1') + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace somedata1 as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "drop table t_heap" + ) + + # Drop old tablespace + node.safe_psql( + "postgres", + "drop tablespace somedata" + ) + + # PAGE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(node, 'somedata1'), + ignore_errors=True) + node.cleanup() + + self.merge_backup(backup_dir, 'node', backup_id) + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + # this compare should fall because we lost some directories + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_merge_tablespaces_1(self): """ Some test here """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + # CREATE NEW TABLESPACE + self.create_tblspace_in_node(node, 'somedata1') + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace somedata1 as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + # PAGE backup + self.backup_node(backup_dir, 'node', node, backup_type="page") + + node.safe_psql( + "postgres", + "drop table t_heap" + ) + node.safe_psql( + "postgres", + "drop tablespace somedata" + ) + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta") + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(node, 'somedata1'), + ignore_errors=True) + node.cleanup() + + self.merge_backup(backup_dir, 'node', backup_id) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) def test_merge_page_truncate(self): """ From e16bdc6660bb5666c485a897e4f444bcdf3d9aab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 13 Dec 2018 13:21:57 +0300 Subject: [PATCH 24/59] tests: add some comments --- tests/merge.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 8ce1795e..2dbe8ee0 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -473,7 +473,10 @@ class MergeTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_merge_tablespaces(self): """ - Create tablespace with table + Create tablespace with table, take FULL backup, + create another tablespace with another table and drop previous + tablespace, take page backup, merge it and restore + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -550,7 +553,10 @@ class MergeTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_merge_tablespaces_1(self): """ - Some test here + Create tablespace with table, take FULL backup, + create another tablespace with another table, take page backup, + drop first tablespace and take delta backup, + merge it and restore """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1202,3 +1208,4 @@ class MergeTest(ProbackupTest, unittest.TestCase): # FULL MERGING # 3. Need new test with corrupted FULL backup +# 4. different compression levels From eec11c074c839bd465780673c417ce7a3f539823 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 13 Dec 2018 17:11:45 +0300 Subject: [PATCH 25/59] Fix comment in main() --- src/pg_probackup.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 04b5d486..ada27e3e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -363,8 +363,9 @@ main(int argc, char *argv[]) } /* - * Read options from env variables or from config file, - * unless we're going to set them via set-config. + * We read options from command line, now we need to read them from + * configuration file since we got backup path and instance name. + * For some commands an instance option isn't required, see above. */ if (instance_name) { From 13c4df812e202bd3c3306c698f3efc493e889a46 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 14 Dec 2018 16:55:32 +0300 Subject: [PATCH 26/59] bugfix: undeclared compress_alg --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ada27e3e..10478970 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -582,7 +582,7 @@ compress_init(void) if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) { #ifndef HAVE_LIBZ - if (compress_alg == ZLIB_COMPRESS) + if (instance_config.compress_alg == ZLIB_COMPRESS) elog(ERROR, "This build does not support zlib compression"); else #endif From 53cb7c1dd4f08ecb1ce1b93aeab03aca1803449e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 14 Dec 2018 18:41:23 +0300 Subject: [PATCH 27/59] Fix removing files which are not in from_backup file list --- src/merge.c | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/src/merge.c b/src/merge.c index b936469b..545a0505 100644 --- a/src/merge.c +++ b/src/merge.c @@ -208,10 +208,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ pgBackupGetPath(to_backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - to_files = dir_read_file_list(from_database_path, /* Use from_database_path - * so root path will be - * equal with 'files' */ - control_file); + to_files = dir_read_file_list(NULL, control_file); /* To delete from leaf, sort in reversed order */ parray_qsort(to_files, pgFileComparePathDesc); /* @@ -219,7 +216,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ pgBackupGetPath(from_backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - files = dir_read_file_list(from_database_path, control_file); + files = dir_read_file_list(NULL, control_file); /* sort by size for load balancing */ parray_qsort(files, pgFileCompareSize); @@ -331,8 +328,18 @@ delete_source_backup: if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { + char to_file_path[MAXPGPATH]; + char *prev_path; + + /* We need full path, file object has relative path */ + join_path_components(to_file_path, to_database_path, file->path); + prev_path = file->path; + file->path = to_file_path; + pgFileDelete(file); elog(VERBOSE, "Deleted \"%s\"", file->path); + + file->path = prev_path; } } @@ -378,13 +385,14 @@ merge_files(void *arg) pgBackup *from_backup = argument->from_backup; int i, num_files = parray_num(argument->files); - int to_root_len = strlen(argument->to_root); for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(argument->files, i); pgFile *to_file; pgFile **res_file; + char from_file_path[MAXPGPATH]; + char *prev_file_path; if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -429,19 +437,23 @@ merge_files(void *arg) continue; } + /* We need to make full path, file object has relative path */ + join_path_components(from_file_path, argument->from_root, file->path); + prev_file_path = file->path; + file->path = from_file_path; + /* * Move the file. We need to decompress it and compress again if * necessary. */ - elog(VERBOSE, "Moving file \"%s\", is_datafile %d, is_cfs %d", + elog(VERBOSE, "Merging file \"%s\", is_datafile %d, is_cfs %d", file->path, file->is_database, file->is_cfs); if (file->is_datafile && !file->is_cfs) { - char to_path_tmp[MAXPGPATH]; /* Path of target file */ + char to_file_path[MAXPGPATH]; /* Path of target file */ - join_path_components(to_path_tmp, argument->to_root, - file->path + to_root_len + 1); + join_path_components(to_file_path, argument->to_root, prev_file_path); /* * We need more complicate algorithm if target file should be @@ -453,7 +465,7 @@ merge_files(void *arg) char tmp_file_path[MAXPGPATH]; char *prev_path; - snprintf(tmp_file_path, MAXPGPATH, "%s_tmp", to_path_tmp); + snprintf(tmp_file_path, MAXPGPATH, "%s_tmp", to_file_path); /* Start the magic */ @@ -479,7 +491,7 @@ merge_files(void *arg) * need the file in directory to_root. */ prev_path = to_file->path; - to_file->path = to_path_tmp; + to_file->path = to_file_path; /* Decompress target file into temporary one */ restore_data_file(tmp_file_path, to_file, false, false, parse_program_version(to_backup->program_version)); @@ -494,7 +506,7 @@ merge_files(void *arg) false, parse_program_version(from_backup->program_version)); - elog(VERBOSE, "Compress file and save it to the directory \"%s\"", + elog(VERBOSE, "Compress file and save it into the directory \"%s\"", argument->to_root); /* Again we need to change path */ @@ -504,7 +516,7 @@ merge_files(void *arg) file->size = pgFileSize(file->path); /* Now we can compress the file */ backup_data_file(NULL, /* We shouldn't need 'arguments' here */ - to_path_tmp, file, + to_file_path, file, to_backup->start_lsn, to_backup->backup_mode, to_backup->compress_alg, @@ -523,7 +535,7 @@ merge_files(void *arg) else { /* We can merge in-place here */ - restore_data_file(to_path_tmp, file, + restore_data_file(to_file_path, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, true, parse_program_version(from_backup->program_version)); @@ -532,8 +544,8 @@ merge_files(void *arg) * We need to calculate write_size, restore_data_file() doesn't * do that. */ - file->write_size = pgFileSize(to_path_tmp); - file->crc = pgFileGetCRC(to_path_tmp, true, true, NULL); + file->write_size = pgFileSize(to_file_path); + file->crc = pgFileGetCRC(to_file_path, true, true, NULL); } } else if (strcmp(file->name, "pg_control") == 0) @@ -548,8 +560,11 @@ merge_files(void *arg) file->compress_alg = to_backup->compress_alg; if (file->write_size != BYTES_INVALID) - elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", + elog(LOG, "Merged file \"%s\": " INT64_FORMAT " bytes", file->path, file->write_size); + + /* Restore relative path */ + file->path = prev_file_path; } /* Data files merging is successful */ From 4650c6f6f80f94e17a5e958ccd868927d7e1653a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 17 Dec 2018 17:44:42 +0300 Subject: [PATCH 28/59] tests: changes for windows compatibility(work in progress) --- tests/helpers/ptrack_helpers.py | 295 ++++++++++++++++---------------- 1 file changed, 143 insertions(+), 152 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 8ea90c24..232daa39 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -122,15 +122,15 @@ def slow_start(self, replica=False): while True: try: self.poll_query_until( - "postgres", - "SELECT not pg_is_in_recovery()") + 'postgres', + 'SELECT not pg_is_in_recovery()') break except Exception as e: continue else: self.poll_query_until( - "postgres", - "SELECT pg_is_in_recovery()") + 'postgres', + 'SELECT pg_is_in_recovery()') # while True: # try: @@ -155,18 +155,18 @@ class ProbackupTest(object): self.test_env = os.environ.copy() envs_list = [ - "LANGUAGE", - "LC_ALL", - "PGCONNECT_TIMEOUT", - "PGDATA", - "PGDATABASE", - "PGHOSTADDR", - "PGREQUIRESSL", - "PGSERVICE", - "PGSSLMODE", - "PGUSER", - "PGPORT", - "PGHOST" + 'LANGUAGE', + 'LC_ALL', + 'PGCONNECT_TIMEOUT', + 'PGDATA', + 'PGDATABASE', + 'PGHOSTADDR', + 'PGREQUIRESSL', + 'PGSERVICE', + 'PGSSLMODE', + 'PGUSER', + 'PGPORT', + 'PGHOST' ] for e in envs_list: @@ -175,8 +175,8 @@ class ProbackupTest(object): except: pass - self.test_env["LC_MESSAGES"] = "C" - self.test_env["LC_TIME"] = "C" + self.test_env['LC_MESSAGES'] = 'C' + self.test_env['LC_TIME'] = 'C' self.paranoia = False if 'PG_PROBACKUP_PARANOIA' in self.test_env: @@ -210,19 +210,19 @@ class ProbackupTest(object): self.user = self.get_username() self.probackup_path = None - if "PGPROBACKUPBIN" in self.test_env: + if 'PGPROBACKUPBIN' in self.test_env: if ( - os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and - os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) + os.path.isfile(self.test_env['PGPROBACKUPBIN']) and + os.access(self.test_env['PGPROBACKUPBIN'], os.X_OK) ): - self.probackup_path = self.test_env["PGPROBACKUPBIN"] + self.probackup_path = self.test_env['PGPROBACKUPBIN'] else: if self.verbose: print('PGPROBACKUPBIN is not an executable file') if not self.probackup_path: probackup_path_tmp = os.path.join( - testgres.get_pg_config()["BINDIR"], 'pg_probackup') + testgres.get_pg_config()['BINDIR'], 'pg_probackup') if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): @@ -233,7 +233,7 @@ class ProbackupTest(object): if not self.probackup_path: probackup_path_tmp = os.path.abspath(os.path.join( - self.dir_path, "../pg_probackup")) + self.dir_path, '../pg_probackup')) if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): @@ -247,16 +247,16 @@ class ProbackupTest(object): exit(1) os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ":" + os.environ['PATH'] + self.probackup_path) + ':' + os.environ['PATH'] self.probackup_old_path = None - if "PGPROBACKUPBIN_OLD" in self.test_env: + if 'PGPROBACKUPBIN_OLD' in self.test_env: if ( - os.path.isfile(self.test_env["PGPROBACKUPBIN_OLD"]) and - os.access(self.test_env["PGPROBACKUPBIN_OLD"], os.X_OK) + os.path.isfile(self.test_env['PGPROBACKUPBIN_OLD']) and + os.access(self.test_env['PGPROBACKUPBIN_OLD'], os.X_OK) ): - self.probackup_old_path = self.test_env["PGPROBACKUPBIN_OLD"] + self.probackup_old_path = self.test_env['PGPROBACKUPBIN_OLD'] else: if self.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') @@ -280,40 +280,37 @@ class ProbackupTest(object): initdb_params=initdb_params, allow_streaming=set_replication) # Sane default parameters - node.append_conf("postgresql.auto.conf", "max_connections = 100") - node.append_conf("postgresql.auto.conf", "shared_buffers = 10MB") - node.append_conf("postgresql.auto.conf", "fsync = on") - node.append_conf("postgresql.auto.conf", "wal_level = logical") - node.append_conf("postgresql.auto.conf", "hot_standby = 'off'") + node.append_conf('postgresql.auto.conf', 'max_connections = 100') + node.append_conf('postgresql.auto.conf', 'shared_buffers = 10MB') + node.append_conf('postgresql.auto.conf', 'fsync = on') + node.append_conf('postgresql.auto.conf', 'wal_level = logical') + node.append_conf('postgresql.auto.conf', 'hot_standby = off') node.append_conf( - "postgresql.auto.conf", "log_line_prefix = '%t [%p]: [%l-1] '") - node.append_conf("postgresql.auto.conf", "log_statement = none") - node.append_conf("postgresql.auto.conf", "log_duration = on") + 'postgresql.auto.conf', "log_line_prefix = '%t [%p]: [%l-1] '") + node.append_conf('postgresql.auto.conf', 'log_statement = none') + node.append_conf('postgresql.auto.conf', 'log_duration = on') node.append_conf( - "postgresql.auto.conf", "log_min_duration_statement = 0") - node.append_conf("postgresql.auto.conf", "log_connections = on") - node.append_conf("postgresql.auto.conf", "log_disconnections = on") + 'postgresql.auto.conf', 'log_min_duration_statement = 0') + node.append_conf('postgresql.auto.conf', 'log_connections = on') + node.append_conf('postgresql.auto.conf', 'log_disconnections = on') # Apply given parameters for key, value in six.iteritems(pg_options): - node.append_conf("postgresql.auto.conf", "%s = %s" % (key, value)) + node.append_conf('postgresql.auto.conf', '%s = %s' % (key, value)) # Allow replication in pg_hba.conf if set_replication: node.append_conf( - "pg_hba.conf", - "local replication all trust\n") - node.append_conf( - "postgresql.auto.conf", - "max_wal_senders = 10") + 'postgresql.auto.conf', + 'max_wal_senders = 10') return node def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): res = node.execute( - "postgres", - "select exists" + 'postgres', + 'select exists' " (select 1 from pg_tablespace where spcname = '{0}')".format( tblspc_name) ) @@ -329,11 +326,11 @@ class ProbackupTest(object): cmd = "CREATE TABLESPACE {0} LOCATION '{1}'".format( tblspc_name, tblspc_path) if cfs: - cmd += " with (compression=true)" + cmd += ' with (compression=true)' if not os.path.exists(tblspc_path): os.makedirs(tblspc_path) - res = node.safe_psql("postgres", cmd) + res = node.safe_psql('postgres', cmd) # Check that tablespace was successfully created # self.assertEqual( # res[0], 0, @@ -344,13 +341,13 @@ class ProbackupTest(object): def get_fork_size(self, node, fork_name): return node.execute( - "postgres", + 'postgres', "select pg_relation_size('{0}')/8192".format(fork_name))[0][0] def get_fork_path(self, node, fork_name): return os.path.join( node.base_dir, 'data', node.execute( - "postgres", + 'postgres', "select pg_relation_filepath('{0}')".format( fork_name))[0][0] ) @@ -378,7 +375,7 @@ class ProbackupTest(object): end_page = pages_per_segment[segment_number] else: file_desc = os.open( - file+".{0}".format(segment_number), os.O_RDONLY + file+'.{0}'.format(segment_number), os.O_RDONLY ) start_page = max(md5_per_page)+1 end_page = end_page + pages_per_segment[segment_number] @@ -481,8 +478,8 @@ class ProbackupTest(object): idx_dict['ptrack'][PageNum]) ) print( - " Old checksumm: {0}\n" - " New checksumm: {1}".format( + ' Old checksumm: {0}\n' + ' New checksumm: {1}'.format( idx_dict['old_pages'][PageNum], idx_dict['new_pages'][PageNum]) ) @@ -545,7 +542,7 @@ class ProbackupTest(object): def run_pb(self, command, async=False, gdb=False, old_binary=False): if not self.probackup_old_path and old_binary: - print("PGPROBACKUPBIN_OLD is not set") + print('PGPROBACKUPBIN_OLD is not set') exit(1) if old_binary: @@ -571,7 +568,7 @@ class ProbackupTest(object): [binary_path] + command, stderr=subprocess.STDOUT, env=self.test_env - ).decode("utf-8") + ).decode('utf-8') if command[0] == 'backup': # return backup ID for line in self.output.splitlines(): @@ -580,7 +577,7 @@ class ProbackupTest(object): else: return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode("utf-8"), self.cmd) + raise ProbackupException(e.output.decode('utf-8'), self.cmd) def run_binary(self, command, async=False): if self.verbose: @@ -599,18 +596,18 @@ class ProbackupTest(object): command, stderr=subprocess.STDOUT, env=self.test_env - ).decode("utf-8") + ).decode('utf-8') return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode("utf-8"), command) + raise ProbackupException(e.output.decode('utf-8'), command) def init_pb(self, backup_dir, old_binary=False): shutil.rmtree(backup_dir, ignore_errors=True) return self.run_pb([ - "init", - "-B", backup_dir + 'init', + '-B', backup_dir ], old_binary=old_binary ) @@ -618,10 +615,10 @@ class ProbackupTest(object): def add_instance(self, backup_dir, instance, node, old_binary=False): return self.run_pb([ - "add-instance", - "--instance={0}".format(instance), - "-B", backup_dir, - "-D", node.data_dir + 'add-instance', + '--instance={0}'.format(instance), + '-B', backup_dir, + '-D', node.data_dir ], old_binary=old_binary ) @@ -629,9 +626,9 @@ class ProbackupTest(object): def del_instance(self, backup_dir, instance, old_binary=False): return self.run_pb([ - "del-instance", - "--instance={0}".format(instance), - "-B", backup_dir + 'del-instance', + '--instance={0}'.format(instance), + '-B', backup_dir ], old_binary=old_binary ) @@ -641,7 +638,7 @@ class ProbackupTest(object): def backup_node( self, backup_dir, instance, node, data_dir=False, - backup_type="full", options=[], async=False, gdb=False, + backup_type='full', options=[], async=False, gdb=False, old_binary=False ): if not node and not data_dir: @@ -655,15 +652,15 @@ class ProbackupTest(object): pgdata = data_dir cmd_list = [ - "backup", - "-B", backup_dir, + 'backup', + '-B', backup_dir, # "-D", pgdata, - "-p", "%i" % node.port, - "-d", "postgres", - "--instance={0}".format(instance) + '-p', '%i' % node.port, + '-d', 'postgres', + '--instance={0}'.format(instance) ] if backup_type: - cmd_list += ["-b", backup_type] + cmd_list += ['-b', backup_type] return self.run_pb(cmd_list + options, async, gdb, old_binary) @@ -671,10 +668,10 @@ class ProbackupTest(object): self, backup_dir, instance, backup_id, async=False, gdb=False, old_binary=False, options=[]): cmd_list = [ - "merge", - "-B", backup_dir, - "--instance={0}".format(instance), - "-i", backup_id + 'merge', + '-B', backup_dir, + '--instance={0}'.format(instance), + '-i', backup_id ] return self.run_pb(cmd_list + options, async, gdb, old_binary) @@ -687,13 +684,13 @@ class ProbackupTest(object): data_dir = node.data_dir cmd_list = [ - "restore", - "-B", backup_dir, - "-D", data_dir, - "--instance={0}".format(instance) + 'restore', + '-B', backup_dir, + '-D', data_dir, + '--instance={0}'.format(instance) ] if backup_id: - cmd_list += ["-i", backup_id] + cmd_list += ['-i', backup_id] return self.run_pb(cmd_list + options, old_binary=old_binary) @@ -705,17 +702,17 @@ class ProbackupTest(object): backup_list = [] specific_record = {} cmd_list = [ - "show", - "-B", backup_dir, + 'show', + '-B', backup_dir, ] if instance: - cmd_list += ["--instance={0}".format(instance)] + cmd_list += ['--instance={0}'.format(instance)] if backup_id: - cmd_list += ["-i", backup_id] + cmd_list += ['-i', backup_id] if as_json: - cmd_list += ["--format=json"] + cmd_list += ['--format=json'] if as_text: # You should print it when calling as_text=true @@ -750,7 +747,7 @@ class ProbackupTest(object): # inverse list so oldest record come first body = body[::-1] # split string in list with string for every header element - header_split = re.split(" +", header) + header_split = re.split(' +', header) # Remove empty items for i in header_split: if i == '': @@ -762,7 +759,7 @@ class ProbackupTest(object): for backup_record in body: backup_record = backup_record.rstrip() # split list with str for every backup record element - backup_record_split = re.split(" +", backup_record) + backup_record_split = re.split(' +', backup_record) # Remove empty items for i in backup_record_split: if i == '': @@ -787,7 +784,7 @@ class ProbackupTest(object): ] # print sanitized_show for line in sanitized_show: - name, var = line.partition(" = ")[::2] + name, var = line.partition(' = ')[::2] var = var.strip('"') var = var.strip("'") specific_record[name.strip()] = var @@ -799,13 +796,13 @@ class ProbackupTest(object): ): cmd_list = [ - "validate", - "-B", backup_dir + 'validate', + '-B', backup_dir ] if instance: - cmd_list += ["--instance={0}".format(instance)] + cmd_list += ['--instance={0}'.format(instance)] if backup_id: - cmd_list += ["-i", backup_id] + cmd_list += ['-i', backup_id] return self.run_pb(cmd_list + options, old_binary=old_binary) @@ -813,48 +810,48 @@ class ProbackupTest(object): self, backup_dir, instance, backup_id=None, options=[], old_binary=False): cmd_list = [ - "delete", - "-B", backup_dir + 'delete', + '-B', backup_dir ] - cmd_list += ["--instance={0}".format(instance)] + cmd_list += ['--instance={0}'.format(instance)] if backup_id: - cmd_list += ["-i", backup_id] + cmd_list += ['-i', backup_id] return self.run_pb(cmd_list + options, old_binary=old_binary) def delete_expired( self, backup_dir, instance, options=[], old_binary=False): cmd_list = [ - "delete", "--expired", "--wal", - "-B", backup_dir, - "--instance={0}".format(instance) + 'delete', '--expired', '--wal', + '-B', backup_dir, + '--instance={0}'.format(instance) ] return self.run_pb(cmd_list + options, old_binary=old_binary) def show_config(self, backup_dir, instance, old_binary=False): out_dict = {} cmd_list = [ - "show-config", - "-B", backup_dir, - "--instance={0}".format(instance) + 'show-config', + '-B', backup_dir, + '--instance={0}'.format(instance) ] res = self.run_pb(cmd_list, old_binary=old_binary).splitlines() for line in res: if not line.startswith('#'): - name, var = line.partition(" = ")[::2] + name, var = line.partition(' = ')[::2] out_dict[name] = var return out_dict def get_recovery_conf(self, node): out_dict = {} with open( - os.path.join(node.data_dir, "recovery.conf"), "r" + os.path.join(node.data_dir, 'recovery.conf'), 'r' ) as recovery_conf: for line in recovery_conf: try: - key, value = line.split("=") + key, value = line.split('=') except: continue out_dict[key.strip()] = value.strip(" '").replace("'\n", "") @@ -870,35 +867,29 @@ class ProbackupTest(object): else: archive_mode = 'on' - # node.append_conf( - # "postgresql.auto.conf", - # "wal_level = archive" - # ) node.append_conf( - "postgresql.auto.conf", - "archive_mode = {0}".format(archive_mode) + 'postgresql.auto.conf', + 'archive_mode = {0}'.format(archive_mode) ) - archive_command = "{0} archive-push -B {1} --instance={2} ".format( + archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( self.probackup_path, backup_dir, instance) + if self.archive_compress or compress: + archive_command = archive_command + '--compress ' + + if overwrite: + archive_command = archive_command + '--overwrite ' + if os.name == 'posix': - if self.archive_compress or compress: - archive_command = archive_command + "--compress " + archive_command = archive_command + '--wal-file-path %p --wal-file-name %f' - if overwrite: - archive_command = archive_command + "--overwrite " - - archive_command = archive_command + "--wal-file-path %p --wal-file-name %f" + elif os.name == 'nt': + archive_command = archive_command + '--wal-file-path "%p" --wal-file-name "%f"' node.append_conf( - "postgresql.auto.conf", + 'postgresql.auto.conf', "archive_command = '{0}'".format( archive_command)) - # elif os.name == 'nt': - # node.append_conf( - # "postgresql.auto.conf", - # "archive_command = 'copy %p {0}\\%f'".format(archive_dir) - # ) def set_replica( self, master, replica, @@ -906,18 +897,18 @@ class ProbackupTest(object): synchronous=False ): replica.append_conf( - "postgresql.auto.conf", "port = {0}".format(replica.port)) + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf('recovery.conf', 'standby_mode = on') replica.append_conf( - "recovery.conf", + 'recovery.conf', "primary_conninfo = 'user={0} port={1} application_name={2}" " sslmode=prefer sslcompression=1'".format( self.user, master.port, replica_name) ) if synchronous: master.append_conf( - "postgresql.auto.conf", + 'postgresql.auto.conf', "synchronous_standby_names='{0}'".format(replica_name) ) master.append_conf( @@ -927,7 +918,7 @@ class ProbackupTest(object): master.reload() def wrong_wal_clean(self, node, wal_size): - wals_dir = os.path.join(self.backup_dir(node), "wal") + wals_dir = os.path.join(self.backup_dir(node), 'wal') wals = [ f for f in os.listdir(wals_dir) if os.path.isfile( os.path.join(wals_dir, f)) @@ -939,25 +930,25 @@ class ProbackupTest(object): def guc_wal_segment_size(self, node): var = node.execute( - "postgres", + 'postgres', "select setting from pg_settings where name = 'wal_segment_size'" ) return int(var[0][0]) * self.guc_wal_block_size(node) def guc_wal_block_size(self, node): var = node.execute( - "postgres", + 'postgres', "select setting from pg_settings where name = 'wal_block_size'" ) return int(var[0][0]) def get_pgpro_edition(self, node): if node.execute( - "postgres", + 'postgres', "select exists (select 1 from" " pg_proc where proname = 'pgpro_edition')" )[0][0]: - var = node.execute("postgres", "select pgpro_edition()") + var = node.execute('postgres', 'select pgpro_edition()') return str(var[0][0]) else: return False @@ -969,9 +960,9 @@ class ProbackupTest(object): def version_to_num(self, version): if not version: return 0 - parts = version.split(".") + parts = version.split('.') while len(parts) < 3: - parts.append("0") + parts.append('0') num = 0 for part in parts: num = num * 100 + int(re.sub("[^\d]", "", part)) @@ -986,25 +977,25 @@ class ProbackupTest(object): """ if isinstance(node, testgres.PostgresNode): if self.version_to_num( - node.safe_psql("postgres", "show server_version") + node.safe_psql('postgres', 'show server_version') ) >= self.version_to_num('10.0'): - node.safe_psql("postgres", "select pg_switch_wal()") + node.safe_psql('postgres', 'select pg_switch_wal()') else: - node.safe_psql("postgres", "select pg_switch_xlog()") + node.safe_psql('postgres', 'select pg_switch_xlog()') else: if self.version_to_num( - node.execute("show server_version")[0][0] + node.execute('show server_version')[0][0] ) >= self.version_to_num('10.0'): - node.execute("select pg_switch_wal()") + node.execute('select pg_switch_wal()') else: - node.execute("select pg_switch_xlog()") + node.execute('select pg_switch_xlog()') def wait_until_replica_catch_with_master(self, master, replica): if self.version_to_num( master.safe_psql( - "postgres", - "show server_version")) >= self.version_to_num('10.0'): + 'postgres', + 'show server_version')) >= self.version_to_num('10.0'): master_function = 'pg_catalog.pg_current_wal_lsn()' replica_function = 'pg_catalog.pg_last_wal_replay_lsn()' else: @@ -1022,7 +1013,7 @@ class ProbackupTest(object): def get_version(self, node): return self.version_to_num( - testgres.get_pg_config()["VERSION"].split(" ")[1]) + testgres.get_pg_config()['VERSION'].split(" ")[1]) def get_bin_path(self, binary): return testgres.get_bin_path(binary) @@ -1217,7 +1208,7 @@ class ProbackupTest(object): host = '127.0.0.1' return psycopg2.connect( - database="postgres", + database='postgres', host='127.0.0.1', port=port, async=True @@ -1233,7 +1224,7 @@ class ProbackupTest(object): elif state == psycopg2.extensions.POLL_READ: select.select([connection.fileno()], [], []) else: - raise psycopg2.OperationalError("poll() returned %s" % state) + raise psycopg2.OperationalError('poll() returned %s' % state) def gdb_attach(self, pid): return GDBobj([str(pid)], self.verbose, attach=True) @@ -1254,7 +1245,7 @@ class GDBobj(ProbackupTest): # Check gdb presense try: gdb_version, _ = subprocess.Popen( - ["gdb", "--version"], + ['gdb', '--version'], stdout=subprocess.PIPE ).communicate() except OSError: From c7d7ceb5bbc0042ed055ab9cf2d2a673d4cb6fb7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 17 Dec 2018 17:54:38 +0300 Subject: [PATCH 29/59] tests: windows compatibility(WIP) --- tests/helpers/ptrack_helpers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 232daa39..51601732 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -7,7 +7,7 @@ import six import testgres import hashlib import re -import pwd +import getpass import select import psycopg2 from time import sleep @@ -89,8 +89,14 @@ def dir_files(base_dir): def is_enterprise(): # pg_config --help + if os.name == 'posix': + cmd = [os.environ['PG_CONFIG'], '--help'] + + elif os.name == 'nt': + cmd = [[os.environ['PG_CONFIG']], ['--help']] + p = subprocess.Popen( - [os.environ['PG_CONFIG'], '--help'], + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -955,7 +961,7 @@ class ProbackupTest(object): def get_username(self): """ Returns current user name """ - return pwd.getpwuid(os.getuid())[0] + return getpass.getuser() def version_to_num(self, version): if not version: From 04b96de31fe5207c9237a549362f3b109b0def7b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 Dec 2018 12:06:04 +0300 Subject: [PATCH 30/59] tests: windows compatibility, archive_command for windows --- tests/helpers/ptrack_helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 51601732..286ce2e8 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -877,8 +877,15 @@ class ProbackupTest(object): 'postgresql.auto.conf', 'archive_mode = {0}'.format(archive_mode) ) - archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( - self.probackup_path, backup_dir, instance) + if os.name == 'posix': + archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( + self.probackup_path, backup_dir, instance) + + elif os.name == 'nt': + archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( + self.probackup_path.replace("\\","\\\\"), + backup_dir.replace("\\","\\\\"), + instance) if self.archive_compress or compress: archive_command = archive_command + '--compress ' From 98c67194a6feda20116a42076d5d9a3c9359b9f8 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 18 Dec 2018 19:00:33 +0300 Subject: [PATCH 31/59] Fixed removing directory for the delete command --- src/delete.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/delete.c b/src/delete.c index 8e9b965f..870fbc42 100644 --- a/src/delete.c +++ b/src/delete.c @@ -296,15 +296,7 @@ delete_backup_files(pgBackup *backup) elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"", i + 1, num_files, file->path); - if (remove(file->path)) - { - if (errno == ENOENT) - elog(VERBOSE, "File \"%s\" is absent", file->path); - else - elog(ERROR, "Cannot remove \"%s\": %s", file->path, - strerror(errno)); - return; - } + pgFileDelete(file); } parray_walk(files, pgFileFree); From 9e8d70d51b35a4744de3132f45f01162cd01862f Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 24 Dec 2018 20:02:50 +0300 Subject: [PATCH 32/59] Fix handling recursive tablespaces. Earlier it was disabled to have recursive tablespaces. The code was buggy, so fix the code and allow to have such tablespaces. --- src/dir.c | 106 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/src/dir.c b/src/dir.c index 386f49bc..96debfa8 100644 --- a/src/dir.c +++ b/src/dir.c @@ -118,7 +118,7 @@ typedef struct TablespaceCreatedList static int BlackListCompare(const void *str1, const void *str2); -static bool dir_check_file(const char *root, pgFile *file); +static char dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, bool omit_symlink, parray *black_list); @@ -448,6 +448,10 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, pgFileFree(file); } +#define CHECK_FALSE 0 +#define CHECK_TRUE 1 +#define CHECK_EXCLUDE_FALSE 2 + /* * Check file or directory. * @@ -456,16 +460,21 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, * Skip files: * - skip temp tables files * - skip unlogged tables files + * Skip recursive tablespace content * Set flags for: * - database directories * - datafiles */ -static bool +static char dir_check_file(const char *root, pgFile *file) { const char *rel_path; int i; int sscanf_res; + bool in_tablespace = false; + + rel_path = GetRelativePath(file->path, root); + in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path); /* Check if we need to exclude file by name */ if (S_ISREG(file->mode)) @@ -478,7 +487,7 @@ dir_check_file(const char *root, pgFile *file) { /* Skip */ elog(VERBOSE, "Excluding file: %s", file->name); - return false; + return CHECK_FALSE; } } @@ -487,14 +496,14 @@ dir_check_file(const char *root, pgFile *file) { /* Skip */ elog(VERBOSE, "Excluding file: %s", file->name); - return false; + return CHECK_FALSE; } } /* * If the directory name is in the exclude list, do not list the * contents. */ - else if (S_ISDIR(file->mode)) + else if (S_ISDIR(file->mode) && !in_tablespace) { /* * If the item in the exclude list starts with '/', compare to @@ -510,20 +519,18 @@ dir_check_file(const char *root, pgFile *file) { elog(VERBOSE, "Excluding directory content: %s", file->name); - return false; + return CHECK_EXCLUDE_FALSE; } } else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) { elog(VERBOSE, "Excluding directory content: %s", file->name); - return false; + return CHECK_EXCLUDE_FALSE; } } } - rel_path = GetRelativePath(file->path, root); - /* * Do not copy tablespaces twice. It may happen if the tablespace is located * inside the PGDATA. @@ -539,14 +546,33 @@ dir_check_file(const char *root, pgFile *file) * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY */ if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) - return false; + return CHECK_FALSE; sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s", &tblspcOid, tmp_rel_path); if (sscanf_res == 0) - return false; + return CHECK_FALSE; } - if (path_is_prefix_of_path("global", rel_path)) + if (in_tablespace) + { + char tmp_rel_path[MAXPGPATH]; + + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", + &(file->tblspcOid), tmp_rel_path, + &(file->dbOid)); + + /* + * We should skip other files and directories rather than + * TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace. + */ + if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0) + return CHECK_FALSE; + + if (sscanf_res == 3 && S_ISDIR(file->mode) && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + file->is_database = true; + } + else if (path_is_prefix_of_path("global", rel_path)) { file->tblspcOid = GLOBALTABLESPACE_OID; @@ -562,22 +588,10 @@ dir_check_file(const char *root, pgFile *file) if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) file->is_database = true; } - else if (path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) - { - char tmp_rel_path[MAXPGPATH]; - - sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", - &(file->tblspcOid), tmp_rel_path, - &(file->dbOid)); - - if (sscanf_res == 3 && S_ISDIR(file->mode) && - strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) - file->is_database = true; - } /* Do not backup ptrack_init files */ if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) - return false; + return CHECK_FALSE; /* * Check files located inside database directories including directory @@ -587,10 +601,10 @@ dir_check_file(const char *root, pgFile *file) file->name && file->name[0]) { if (strcmp(file->name, "pg_internal.init") == 0) - return false; + return CHECK_FALSE; /* Do not backup temp files */ else if (file->name[0] == 't' && isdigit(file->name[1])) - return false; + return CHECK_FALSE; else if (isdigit(file->name[0])) { char *fork_name; @@ -605,14 +619,14 @@ dir_check_file(const char *root, pgFile *file) /* Do not backup ptrack files */ if (strcmp(file->forkName, "ptrack") == 0) - return false; + return CHECK_FALSE; } else { len = strlen(file->name); /* reloid.cfm */ if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) - return true; + return CHECK_TRUE; sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), &(file->segno), suffix); @@ -624,7 +638,7 @@ dir_check_file(const char *root, pgFile *file) } } - return true; + return CHECK_TRUE; } /* @@ -659,6 +673,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, { pgFile *file; char child[MAXPGPATH]; + char check_res; join_path_components(child, parent->path, dent->d_name); @@ -694,21 +709,24 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, continue; } - /* We add the directory anyway */ - if (S_ISDIR(file->mode)) - parray_append(files, file); - - if (exclude && !dir_check_file(root, file)) + if (exclude) { - if (S_ISREG(file->mode)) + check_res = dir_check_file(root, file); + if (check_res == CHECK_FALSE) + { + /* Skip */ pgFileFree(file); - /* Skip */ - continue; + continue; + } + else if (check_res == CHECK_EXCLUDE_FALSE) + { + /* We add the directory itself which content was excluded */ + parray_append(files, file); + continue; + } } - /* At least add the file */ - if (S_ISREG(file->mode)) - parray_append(files, file); + parray_append(files, file); /* * If the entry is a directory call dir_list_file_internal() @@ -1219,11 +1237,7 @@ print_file_list(FILE *out, const parray *files, const char *root) if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); -#ifndef WIN32 - if (S_ISLNK(file->mode)) -#else - if (pgwin32_is_junction(file->path)) -#endif + if (file->linked) fprintf(out, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != BLOCKNUM_INVALID) From 31830df36d73b3e68645495ba7310d18eb6404a5 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 25 Dec 2018 14:08:15 +0300 Subject: [PATCH 33/59] Added initialization of variable for tablespace renaming --- src/dir.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dir.c b/src/dir.c index 96debfa8..3eeb96f3 100644 --- a/src/dir.c +++ b/src/dir.c @@ -894,6 +894,7 @@ opt_tablespace_map(ConfigOption *opt, const char *arg) char *dst_ptr; const char *arg_ptr; + memset(cell, 0, sizeof(TablespaceListCell)); dst_ptr = dst = cell->old_dir; for (arg_ptr = arg; *arg_ptr; arg_ptr++) { From a42895620d5b183969b946af0fb46729b05e99e4 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 25 Dec 2018 14:29:47 +0300 Subject: [PATCH 34/59] Fix filling and releasing of black_list --- src/dir.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index 3eeb96f3..37f41ddd 100644 --- a/src/dir.c +++ b/src/dir.c @@ -415,6 +415,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, while (fgets(buf, lengthof(buf), black_list_file) != NULL) { + black_item[0] = '\0'; join_path_components(black_item, instance_config.pgdata, buf); if (black_item[strlen(black_item) - 1] == '\n') @@ -423,7 +424,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (black_item[0] == '#' || black_item[0] == '\0') continue; - parray_append(black_list, black_item); + parray_append(black_list, pgut_strdup(black_item)); } fclose(black_list_file); @@ -446,6 +447,12 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (!add_root) pgFileFree(file); + + if (black_list) + { + parray_walk(black_list, pfree); + parray_free(black_list); + } } #define CHECK_FALSE 0 From 02860600d9df5e6686e3a1d63a60b781c2a552ef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 16:44:37 +0300 Subject: [PATCH 35/59] bugfix: elog fact of copying file after locking it, not before --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 7046d1d0..845ff03e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2216,9 +2216,9 @@ backup_files(void *arg) struct stat buf; pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - elog(VERBOSE, "Copying file: \"%s\" ", file->path); if (!pg_atomic_test_set_flag(&file->lock)) continue; + elog(VERBOSE, "Copying file: \"%s\" ", file->path); /* check for interrupt */ if (interrupted) From f8340f5eb8898e598d3cdcc6fde40879b6f9e127 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 17:12:48 +0300 Subject: [PATCH 36/59] tests: adapt test_tablespace_in_pgdata_pgpro_1376 to 9e8d70d51b35 --- tests/backup_test.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 9c8a0955..e6d9a336 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -459,22 +459,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1000) i") - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of too many levels " - "of symbolic linking\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Too many levels of symbolic links' in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + backup_id_1 = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) node.safe_psql( "postgres", @@ -495,7 +482,8 @@ class BackupTest(ProbackupTest, unittest.TestCase): ).rstrip() list = [] - for root, dirs, files in os.walk(backup_dir): + for root, dirs, files in os.walk(os.path.join( + backup_dir, 'backups/node', backup_id_1)): for file in files: if file == relfilenode: path = os.path.join(root, file) From 7b66795b8a2ed6abf31c4c1c11195db8e0e76613 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 17:48:49 +0300 Subject: [PATCH 37/59] tests: windows support --- tests/archive.py | 4 +-- tests/auth_test.py | 35 ++++++++++--------- tests/backup_test.py | 20 +++++------ tests/cfs_backup.py | 10 +++--- tests/cfs_restore.py | 2 +- tests/compression.py | 10 +++--- tests/delete_test.py | 8 ++--- tests/delta.py | 56 +++++++++++++++--------------- tests/exclude.py | 4 +-- tests/false_positive.py | 10 +++--- tests/helpers/ptrack_helpers.py | 6 ++-- tests/merge.py | 22 ++++++------ tests/page.py | 34 +++++++++--------- tests/pgpro560.py | 6 ++-- tests/pgpro589.py | 2 +- tests/ptrack_clean.py | 6 ++-- tests/ptrack_cluster.py | 12 +++---- tests/ptrack_empty.py | 6 ++-- tests/ptrack_move_to_tablespace.py | 2 +- tests/ptrack_recovery.py | 4 +-- tests/replica.py | 2 +- tests/restore_test.py | 44 +++++++++++------------ tests/retention_test.py | 6 ++-- tests/show_test.py | 12 +++---- tests/validate_test.py | 48 ++++++++++++------------- 25 files changed, 186 insertions(+), 185 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 180b7ad6..45e735e8 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -732,7 +732,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() if self.get_version(node) < 100000: pg_receivexlog_path = self.get_bin_path('pg_receivexlog') else: @@ -806,7 +806,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() if self.get_version(node) < self.version_to_num('10.0'): return unittest.skip('You need PostgreSQL 10 for this test') else: diff --git a/tests/auth_test.py b/tests/auth_test.py index fc21a480..c5f1ae33 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -43,7 +43,7 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") @@ -203,25 +203,26 @@ class AuthTest(unittest.TestCase): cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) try: - cls.node.start() + cls.node.slow_start() except StartNodeException: raise unittest.skip("Node hasn't started") - cls.node.safe_psql("postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; \ - GRANT USAGE ON SCHEMA pg_catalog TO backup; \ - GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; \ - GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; \ - GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; \ - GRANT EXECUTE ON FUNCTION txid_current() TO backup; \ - GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; \ - GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; \ - GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') @classmethod diff --git a/tests/backup_test.py b/tests/backup_test.py index e6d9a336..08abd971 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -27,7 +27,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] @@ -97,7 +97,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, @@ -121,7 +121,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() try: self.backup_node(backup_dir, 'node', node, backup_type="page") @@ -177,7 +177,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) file = os.path.join( @@ -241,7 +241,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, @@ -272,7 +272,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, backup_type="full", @@ -301,7 +301,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, @@ -358,7 +358,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, backup_type="full", @@ -383,7 +383,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): f.write(b"bla") f.flush() f.close - node.start() + node.slow_start() try: self.backup_node( @@ -431,7 +431,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node( node, 'tblspace1', diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 41232032..76d4a0a1 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -34,7 +34,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.add_instance(self.backup_dir, 'node', self.node) self.set_archiving(self.backup_dir, 'node', self.node) - self.node.start() + self.node.slow_start() self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) @@ -745,7 +745,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.restore_node( self.backup_dir, 'node', self.node, backup_id=backup_id_full, options=["-j", "4"]) - self.node.start() + self.node.slow_start() self.assertEqual( full_result, self.node.safe_psql("postgres", "SELECT * FROM t_heap"), @@ -760,7 +760,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.restore_node( self.backup_dir, 'node', self.node, backup_id=backup_id_page, options=["-j", "4"]) - self.node.start() + self.node.slow_start() self.assertEqual( page_result, self.node.safe_psql("postgres", "SELECT * FROM t_heap"), @@ -879,7 +879,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.restore_node( self.backup_dir, 'node', self.node, backup_id=backup_id_full, options=["-j", "4"]) - self.node.start() + self.node.slow_start() self.assertEqual( full_result_1, self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), @@ -905,7 +905,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.restore_node( self.backup_dir, 'node', self.node, backup_id=backup_id_page, options=["-j", "4"]) - self.node.start() + self.node.slow_start() self.assertEqual( page_result_1, self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 1aefef89..a18d0903 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -43,7 +43,7 @@ class CfsRestoreBase(ProbackupTest, unittest.TestCase): self.add_instance(self.backup_dir, 'node', self.node) self.set_archiving(self.backup_dir, 'node', self.node) - self.node.start() + self.node.slow_start() self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) self.add_data_in_cluster() diff --git a/tests/compression.py b/tests/compression.py index 2e712a15..f2c8614e 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -30,7 +30,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -142,7 +142,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -250,7 +250,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -360,7 +360,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -471,7 +471,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() try: self.backup_node( diff --git a/tests/delete_test.py b/tests/delete_test.py index b36c2c92..f58fc5fd 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -24,7 +24,7 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # full backup self.backup_node(backup_dir, 'node', node) @@ -72,7 +72,7 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # full backup mode self.backup_node(backup_dir, 'node', node) @@ -112,7 +112,7 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # full backup mode self.backup_node(backup_dir, 'node', node) @@ -152,7 +152,7 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", diff --git a/tests/delta.py b/tests/delta.py index 55cc03be..76673ceb 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -41,7 +41,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -89,7 +89,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -123,7 +123,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -180,7 +180,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -214,7 +214,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -252,7 +252,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -279,7 +279,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -318,7 +318,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -334,7 +334,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() @@ -365,7 +365,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) # self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -401,7 +401,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -417,7 +417,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() @@ -451,7 +451,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) # self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -533,7 +533,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -599,7 +599,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -627,7 +627,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -681,7 +681,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # DROP DATABASE DB1 node.safe_psql( @@ -716,7 +716,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() try: node_restored.safe_psql('db1', 'select 1') @@ -761,7 +761,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -842,7 +842,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -868,7 +868,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP self.create_tblspace_in_node(node, 'somedata') @@ -961,7 +961,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup @@ -1029,7 +1029,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -1056,7 +1056,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -1115,7 +1115,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -1134,7 +1134,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, @@ -1193,7 +1193,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, backup_type="full", @@ -1218,7 +1218,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): f.write(b"bla") f.flush() f.close - node.start() + node.slow_start() try: self.backup_node( diff --git a/tests/exclude.py b/tests/exclude.py index 3fd3341f..30e500d6 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -28,7 +28,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() conn = node.connect() with node.connect("postgres") as conn: @@ -122,7 +122,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() conn = node.connect() with node.connect("postgres") as conn: diff --git a/tests/false_positive.py b/tests/false_positive.py index df7b1334..24f02276 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -28,7 +28,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -66,7 +66,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) file = os.path.join( @@ -132,7 +132,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -216,7 +216,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -308,7 +308,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 286ce2e8..43ecaedf 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -218,10 +218,10 @@ class ProbackupTest(object): self.probackup_path = None if 'PGPROBACKUPBIN' in self.test_env: if ( - os.path.isfile(self.test_env['PGPROBACKUPBIN']) and - os.access(self.test_env['PGPROBACKUPBIN'], os.X_OK) + os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and + os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) ): - self.probackup_path = self.test_env['PGPROBACKUPBIN'] + self.probackup_path = self.test_env["PGPROBACKUPBIN"] else: if self.verbose: print('PGPROBACKUPBIN is not an executable file') diff --git a/tests/merge.py b/tests/merge.py index 2dbe8ee0..6b09c373 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -26,7 +26,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) self.set_archiving(backup_dir, "node", node) - node.start() + node.slow_start() # Do full backup self.backup_node(backup_dir, "node", node) @@ -115,7 +115,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) self.set_archiving(backup_dir, "node", node) - node.start() + node.slow_start() # Do full compressed backup self.backup_node(backup_dir, "node", node, options=[ @@ -493,7 +493,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -573,7 +573,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -662,7 +662,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -757,7 +757,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -853,7 +853,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -944,7 +944,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -1006,7 +1006,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -1029,7 +1029,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL backup self.backup_node(backup_dir, 'node', node) @@ -1107,7 +1107,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL backup self.backup_node(backup_dir, 'node', node) diff --git a/tests/page.py b/tests/page.py index d0ebbb9b..9e23ecaa 100644 --- a/tests/page.py +++ b/tests/page.py @@ -40,7 +40,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -128,7 +128,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -217,7 +217,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -317,7 +317,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -397,7 +397,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup @@ -449,7 +449,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -477,7 +477,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -530,7 +530,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -558,7 +558,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) node_restored.cleanup() self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # Do full backup self.backup_node(backup_dir, 'node', node) @@ -598,7 +598,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Check restored node count2 = node_restored.execute("postgres", "select count(*) from test") @@ -627,7 +627,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # Do full backup self.backup_node(backup_dir, 'node', node) @@ -656,7 +656,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # Drop node and restore it node.cleanup() self.restore_node(backup_dir, 'node', node) - node.start() + node.slow_start() # Clean after yourself node.cleanup() @@ -681,7 +681,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -771,7 +771,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -896,11 +896,11 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.add_instance(backup_dir, 'alien_node', alien_node) self.set_archiving(backup_dir, 'alien_node', alien_node) - alien_node.start() + alien_node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'alien_node', alien_node) @@ -1013,7 +1013,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index bf334556..6c2b8863 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -28,7 +28,7 @@ class CheckSystemID(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() file = os.path.join(node.base_dir,'data', 'global', 'pg_control') os.remove(file) @@ -60,13 +60,13 @@ class CheckSystemID(ProbackupTest, unittest.TestCase): initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - node1.start() + node1.slow_start() node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - node2.start() + node2.slow_start() backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/pgpro589.py b/tests/pgpro589.py index bd40f16d..d3bb9d90 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -30,7 +30,7 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): # make erroneus archive_command node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") - node.start() + node.slow_start() node.pgbench_init(scale=5) pgbench = node.pgbench( diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 2b122f2f..605d631a 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -26,7 +26,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -136,7 +136,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -149,7 +149,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start() # Create table and indexes master.safe_psql( diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index 2fdfe097..41de27df 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -26,7 +26,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -107,7 +107,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() # Create table and indexes node.safe_psql( @@ -185,7 +185,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -198,7 +198,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start() # Create table and indexes master.safe_psql( @@ -285,7 +285,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -298,7 +298,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start() # Create table and indexes master.safe_psql( diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index f20362b1..18f289e7 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -27,7 +27,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -103,7 +103,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -116,7 +116,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start() # Create table master.safe_psql( diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index 95a7a5aa..41b92cf7 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -24,7 +24,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index 1a6607c9..5889b06e 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -25,7 +25,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -56,7 +56,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): print('Killing postmaster. Losing Ptrack changes') node.stop(['-m', 'immediate', '-D', node.data_dir]) if not node.status(): - node.start() + node.slow_start() else: print("Die! Die! Why won't you die?... Why won't you die?") exit(1) diff --git a/tests/replica.py b/tests/replica.py index 4fbd08e6..052dee2c 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -30,7 +30,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): 'max_wal_senders': '2', 'ptrack_enable': 'on'} ) - master.start() + master.slow_start() self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) diff --git a/tests/restore_test.py b/tests/restore_test.py index 0afcca8a..2ada3955 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -27,7 +27,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) pgbench = node.pgbench( @@ -74,7 +74,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -122,7 +122,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -192,7 +192,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.append_conf("postgresql.auto.conf", "TimeZone = Europe/Moscow") - node.start() + node.slow_start() node.pgbench_init(scale=2) before = node.execute("postgres", "SELECT * FROM pgbench_branches") @@ -240,7 +240,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) with node.connect("postgres") as con: @@ -304,7 +304,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) with node.connect("postgres") as con: @@ -371,7 +371,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) with node.connect("postgres") as con: @@ -444,7 +444,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) with node.connect("postgres") as con: @@ -513,7 +513,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -561,7 +561,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -620,7 +620,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -675,7 +675,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -741,7 +741,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # wal_segment_size = self.guc_wal_segment_size(node) node.pgbench_init(scale=2) @@ -805,7 +805,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # Create tablespace tblspc_path = os.path.join(node.base_dir, "tblspc") @@ -921,7 +921,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # Full backup self.backup_node(backup_dir, 'node', node) @@ -1004,7 +1004,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1054,7 +1054,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1103,7 +1103,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1151,7 +1151,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) if self.paranoia: @@ -1206,7 +1206,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -1255,7 +1255,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -1336,7 +1336,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) diff --git a/tests/retention_test.py b/tests/retention_test.py index 652f7c39..91fef215 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -23,7 +23,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() with open(os.path.join( backup_dir, 'backups', 'node', @@ -80,7 +80,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() with open( os.path.join( @@ -134,7 +134,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", diff --git a/tests/show_test.py b/tests/show_test.py index 50ad5be1..655c0b8e 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -23,7 +23,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.assertEqual( self.backup_node( @@ -51,7 +51,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.assertEqual( self.backup_node( @@ -79,7 +79,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -125,7 +125,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -154,7 +154,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -185,7 +185,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) diff --git a/tests/validate_test.py b/tests/validate_test.py index a7f8d437..6153a4a4 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -221,7 +221,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -306,7 +306,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -429,7 +429,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -628,7 +628,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -812,7 +812,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -960,7 +960,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -1046,7 +1046,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -1121,7 +1121,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -1183,7 +1183,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") @@ -1255,7 +1255,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=3) @@ -1333,7 +1333,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -1427,7 +1427,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1470,7 +1470,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) recovery_time = self.show_pb( @@ -1500,7 +1500,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) self.set_archiving(backup_dir, 'node1', node1) - node1.start() + node1.slow_start() backup_id = self.backup_node( backup_dir, 'node1', node1, options=["--stream"]) @@ -1594,7 +1594,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -1717,7 +1717,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -1829,7 +1829,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -2179,7 +2179,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -2405,7 +2405,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -2481,7 +2481,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # CHAIN1 self.backup_node(backup_dir, 'node', node) @@ -2623,7 +2623,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # CHAIN1 self.backup_node(backup_dir, 'node', node) @@ -2744,7 +2744,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # CHAIN1 self.backup_node(backup_dir, 'node', node) @@ -2932,7 +2932,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # CHAIN1 self.backup_node(backup_dir, 'node', node) @@ -3080,7 +3080,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) From 9a632f0f568b83d4dd94cffa7d841c09c4f0b37f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 22:04:56 +0300 Subject: [PATCH 38/59] tests: windows support, fix test_incremental_backup_corrupt_full() --- tests/backup_test.py | 10 ++++++---- tests/helpers/ptrack_helpers.py | 9 +++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 08abd971..4c447f08 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -195,12 +195,14 @@ class BackupTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "INFO: Validate backups of the instance 'node'\n" in e.message and - "WARNING: Backup file \"{0}\" is not found\n".format( + "INFO: Validate backups of the instance 'node'" in e.message and + "WARNING: Backup file".format( file) in e.message and - "WARNING: Backup {0} data files are corrupted\n".format( + "is not found".format(file) in e.message and + "{0}\" is not found".format(file) in e.message and + "WARNING: Backup {0} data files are corrupted".format( backup_id) in e.message and - "WARNING: Some backups are not valid\n" in e.message, + "WARNING: Some backups are not valid" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 43ecaedf..17fc4f23 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -252,8 +252,13 @@ class ProbackupTest(object): print('pg_probackup binary is not found') exit(1) - os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ':' + os.environ['PATH'] + if os.name == 'posix': + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ':' + os.environ['PATH'] + + elif os.name == 'nt': + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ';' + os.environ['PATH'] self.probackup_old_path = None From c1fd7ce031f76f30cdee97e98bb09ff84d583843 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 22:07:54 +0300 Subject: [PATCH 39/59] minor fix --- tests/backup_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 4c447f08..08f397e1 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -199,7 +199,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): "WARNING: Backup file".format( file) in e.message and "is not found".format(file) in e.message and - "{0}\" is not found".format(file) in e.message and "WARNING: Backup {0} data files are corrupted".format( backup_id) in e.message and "WARNING: Some backups are not valid" in e.message, From 800ba045f508f19c69b5a64a37ed4ccce74ee96e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 26 Dec 2018 15:00:05 +0300 Subject: [PATCH 40/59] No need to check static array backup->program_version to NULL --- src/validate.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/validate.c b/src/validate.c index 14d347ca..a283593e 100644 --- a/src/validate.c +++ b/src/validate.c @@ -53,8 +53,7 @@ pgBackupValidate(pgBackup *backup) int i; /* Check backup version */ - if (backup->program_version && - parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) + if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " "Please upgrade pg_probackup binary.", From b61df02ce168de07e30124187f07a7c885cad797 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 26 Dec 2018 22:59:13 +0300 Subject: [PATCH 41/59] tests: windows compatibility, cross-platform path for base_dir --- tests/archive.py | 28 ++++++------ tests/auth_test.py | 2 +- tests/backup_test.py | 18 ++++---- tests/compatibility.py | 16 +++---- tests/compression.py | 12 ++--- tests/delete_test.py | 27 ++++++----- tests/delta.py | 58 ++++++++++++------------ tests/exclude.py | 8 ++-- tests/false_positive.py | 23 +++++----- tests/init_test.py | 6 +-- tests/merge.py | 36 +++++++-------- tests/option_test.py | 4 +- tests/page.py | 41 +++++++++-------- tests/pgpro560.py | 21 +++++---- tests/pgpro589.py | 2 +- tests/ptrack.py | 62 +++++++++++++------------- tests/ptrack_clean.py | 6 +-- tests/ptrack_cluster.py | 12 ++--- tests/ptrack_empty.py | 10 ++--- tests/ptrack_move_to_tablespace.py | 2 +- tests/ptrack_recovery.py | 2 +- tests/ptrack_truncate.py | 6 +-- tests/ptrack_vacuum.py | 6 +-- tests/ptrack_vacuum_bits_frozen.py | 6 +-- tests/ptrack_vacuum_bits_visibility.py | 2 +- tests/ptrack_vacuum_full.py | 6 +-- tests/ptrack_vacuum_truncate.py | 6 +-- tests/replica.py | 21 ++++----- tests/restore_test.py | 48 ++++++++++---------- tests/retention_test.py | 6 +-- tests/show_test.py | 12 ++--- tests/validate_test.py | 54 +++++++++++----------- 32 files changed, 287 insertions(+), 282 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 45e735e8..1d0d7af5 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -21,7 +21,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -79,7 +79,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -228,7 +228,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -283,7 +283,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -352,7 +352,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -418,7 +418,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -432,7 +432,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): master.slow_start() replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() master.psql( @@ -481,7 +481,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # RESTORE FULL BACKUP TAKEN FROM replica node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( @@ -550,14 +550,14 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s'} ) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.init_pb(backup_dir) @@ -640,7 +640,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -648,7 +648,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): 'archive_timeout': '10s'} ) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.init_pb(backup_dir) @@ -723,7 +723,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -797,7 +797,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/auth_test.py b/tests/auth_test.py index c5f1ae33..cb13e607 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -32,7 +32,7 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/backup_test.py b/tests/backup_test.py index 08f397e1..24cd7b36 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -17,7 +17,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """standart backup modes with ARCHIVE WAL method""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -89,7 +89,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """full backup with smooth checkpoint""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -113,7 +113,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """page-level backup without validated full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -169,7 +169,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """page-level backup with corrupted full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -234,7 +234,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """ptrack multi thread backup mode""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -262,7 +262,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """ptrack multi thread backup mode and stream""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -293,7 +293,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -350,7 +350,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -423,7 +423,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): """PGPRO-1376 """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} diff --git a/tests/compatibility.py b/tests/compatibility.py index 39070b3f..d8e1137a 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -16,7 +16,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -47,7 +47,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -116,7 +116,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -147,7 +147,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -216,7 +216,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -247,7 +247,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -316,7 +316,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -342,7 +342,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): # restore OLD FULL with new binary node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() diff --git a/tests/compression.py b/tests/compression.py index f2c8614e..2cb44018 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -18,7 +18,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -131,7 +131,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -238,7 +238,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -348,7 +348,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -458,7 +458,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -503,7 +503,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/delete_test.py b/tests/delete_test.py index f58fc5fd..f49c01bf 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -16,10 +16,10 @@ class DeleteTest(ProbackupTest, unittest.TestCase): """delete full backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -64,10 +64,11 @@ class DeleteTest(ProbackupTest, unittest.TestCase): def test_delete_increment_page(self): """delete increment and all after him""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -104,10 +105,11 @@ class DeleteTest(ProbackupTest, unittest.TestCase): def test_delete_increment_ptrack(self): """delete increment and all after him""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} - ) + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -144,10 +146,11 @@ class DeleteTest(ProbackupTest, unittest.TestCase): def test_delete_orphaned_wal_segments(self): """make archive node, make three full backups, delete second backup without --wal option, then delete orphaned wals via --wal option""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/delta.py b/tests/delta.py index 76673ceb..d51c374c 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -23,19 +23,17 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) + node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -105,7 +103,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -116,7 +114,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -196,7 +194,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -207,7 +205,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -266,7 +264,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -352,7 +350,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -434,7 +432,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -478,7 +476,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # RESTORE NODE restored_node = self.make_simple_node( - base_dir="{0}/{1}/restored_node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -517,7 +515,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -527,7 +525,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -613,7 +611,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -659,7 +657,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -746,7 +744,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -820,7 +818,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -856,7 +854,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -901,7 +899,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -949,7 +947,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -1002,7 +1000,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1043,7 +1041,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -1092,7 +1090,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1125,7 +1123,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1184,7 +1182,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1259,7 +1257,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1307,7 +1305,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # Restore DELTA backup node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) node_restored.cleanup() diff --git a/tests/exclude.py b/tests/exclude.py index 30e500d6..b290a3b6 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -18,7 +18,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -109,7 +109,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -149,8 +149,8 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() self.restore_node( diff --git a/tests/false_positive.py b/tests/false_positive.py index 24f02276..e67b7598 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -18,12 +18,12 @@ class FalsePositive(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2'} - ) + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -58,10 +58,10 @@ class FalsePositive(ProbackupTest, unittest.TestCase): """page-level backup with corrupted full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'} - ) + pg_options={'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -120,7 +120,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -204,7 +204,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -300,10 +300,11 @@ class FalsePositive(ProbackupTest, unittest.TestCase): def test_multiple_delete(self): """delete multiple backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/init_test.py b/tests/init_test.py index 0b91dafa..ff416e24 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -14,7 +14,7 @@ class InitTest(ProbackupTest, unittest.TestCase): """Success normal init""" fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) self.init_pb(backup_dir) self.assertEqual( dir_files(backup_dir), @@ -66,7 +66,7 @@ class InitTest(ProbackupTest, unittest.TestCase): """Failure with backup catalog already existed""" fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) self.init_pb(backup_dir) try: self.show_pb(backup_dir, 'node') @@ -85,7 +85,7 @@ class InitTest(ProbackupTest, unittest.TestCase): """failure with backup catalog should be given as absolute path""" fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) try: self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( diff --git a/tests/merge.py b/tests/merge.py index 6b09c373..dc5e3949 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -19,7 +19,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"] ) @@ -108,7 +108,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"] ) @@ -173,7 +173,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' @@ -250,7 +250,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' @@ -326,7 +326,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' @@ -403,7 +403,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' @@ -481,7 +481,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -561,7 +561,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -645,7 +645,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -656,7 +656,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -740,7 +740,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -751,7 +751,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -835,7 +835,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -847,7 +847,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -931,7 +931,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -983,7 +983,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1019,7 +1019,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica' @@ -1097,7 +1097,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica' diff --git a/tests/option_test.py b/tests/option_test.py index faed430a..5a64a8b6 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -52,7 +52,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -118,7 +118,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), pg_options={ 'wal_level': 'logical', 'max_wal_senders': '2'}) diff --git a/tests/page.py b/tests/page.py index 9e23ecaa..388b0df9 100644 --- a/tests/page.py +++ b/tests/page.py @@ -23,7 +23,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -34,7 +34,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -116,7 +116,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -205,7 +205,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -300,7 +300,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -341,7 +341,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # RESTORE NODE restored_node = self.make_simple_node( - base_dir="{0}/{1}/restored_node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -384,7 +384,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -427,7 +427,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -464,7 +464,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -508,7 +508,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -544,14 +544,14 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ "hot_standby": "on" } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -619,7 +619,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={} ) @@ -673,7 +673,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -763,7 +763,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -884,13 +884,12 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) alien_node = self.make_simple_node( - base_dir="{0}/{1}/alien_node".format(module_name, fname) - ) + base_dir=os.path.join(module_name, fname, 'alien_node')) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1004,7 +1003,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1042,7 +1041,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1082,7 +1081,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() diff --git a/tests/pgpro560.py b/tests/pgpro560.py index 6c2b8863..ed011074 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -20,11 +20,12 @@ class CheckSystemID(ProbackupTest, unittest.TestCase): check that backup failed """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -55,17 +56,19 @@ class CheckSystemID(ProbackupTest, unittest.TestCase): check that backup failed """ fname = self.id().split('.')[3] - node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(module_name, fname), + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + node1.slow_start() - node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(module_name, fname), + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + node2.slow_start() backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/pgpro589.py b/tests/pgpro589.py index d3bb9d90..11809528 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -19,7 +19,7 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/ptrack.py b/tests/ptrack.py index 5d01d882..376e36a9 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -21,7 +21,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -70,7 +70,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -128,7 +128,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -139,7 +139,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -190,7 +190,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -201,7 +201,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -286,7 +286,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -298,7 +298,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -372,7 +372,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -383,7 +383,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -448,7 +448,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -520,7 +520,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -616,7 +616,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -722,7 +722,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -806,7 +806,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -878,7 +878,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -955,7 +955,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1003,7 +1003,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1082,7 +1082,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1106,7 +1106,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.backup_node( @@ -1159,7 +1159,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1183,7 +1183,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -1232,7 +1232,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1280,7 +1280,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1317,7 +1317,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( backup_dir, 'node', @@ -1351,7 +1351,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1435,7 +1435,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1478,7 +1478,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Restore ptrack backup restored_node = self.make_simple_node( - base_dir="{0}/{1}/restored_node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'restored_node')) restored_node.cleanup() tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') @@ -1548,7 +1548,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1613,7 +1613,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # RESTORE NODE restored_node = self.make_simple_node( - base_dir="{0}/{1}/restored_node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -1655,7 +1655,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 605d631a..01d364e9 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -15,7 +15,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): """Take backups of every available types and check that PTRACK is clean""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -124,7 +124,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): """Take backups of every available types from master and check that PTRACK on replica is clean""" fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -141,7 +141,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index 41de27df..270cb1b4 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -15,7 +15,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_cluster_on_btree(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -96,7 +96,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_cluster_on_gist(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -174,7 +174,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_cluster_on_btree_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -190,7 +190,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -274,7 +274,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_cluster_on_gist_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -290,7 +290,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index 18f289e7..578c8a73 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -15,7 +15,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): """Take backups of every available types and check that PTRACK is clean""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -58,7 +58,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): node.safe_psql('postgres', 'checkpoint') node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() tblspace1 = self.get_tblspace_path(node, 'somedata') @@ -93,7 +93,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): """Take backups of every available types from master and check that PTRACK on replica is clean""" fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -108,7 +108,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -151,7 +151,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.wait_until_replica_catch_with_master(master, replica) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() # Take PTRACK backup diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index 41b92cf7..2011d2a2 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_recovery(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index 5889b06e..068d7dd0 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -14,7 +14,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_recovery(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py index 662a93f8..45a431e0 100644 --- a/tests/ptrack_truncate.py +++ b/tests/ptrack_truncate.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_truncate(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -74,7 +74,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_truncate_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -91,7 +91,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index 2283c740..bf3d8d69 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -100,7 +100,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -117,7 +117,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index 0da323b9..637807b5 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_bits_frozen(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -93,7 +93,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_bits_frozen_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -109,7 +109,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py index a5fa5206..53be4401 100644 --- a/tests/ptrack_vacuum_bits_visibility.py +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_bits_visibility(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index a20ae164..826742fb 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -14,7 +14,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_full(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -95,7 +95,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_full_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -112,7 +112,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 406e2b3a..c0991353 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_truncate(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -94,7 +94,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_truncate_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -111,7 +111,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/replica.py b/tests/replica.py index 052dee2c..d9eabcbb 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -22,7 +22,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -45,13 +45,14 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # take full backup and restore it self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica) # Check data correctness on replica replica.slow_start(replica=True) + exit(1) after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -79,7 +80,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -138,7 +139,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -152,7 +153,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): master.slow_start() replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -211,7 +212,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # RESTORE FULL BACKUP TAKEN FROM replica node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -289,7 +290,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -305,7 +306,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): master.slow_start() replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -350,7 +351,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -364,7 +365,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): master.slow_start() replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) diff --git a/tests/restore_test.py b/tests/restore_test.py index 2ada3955..92c85a79 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -18,7 +18,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to latest from full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -66,7 +66,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to latest from full + page backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -114,7 +114,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to target timeline""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -183,7 +183,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to target time""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -232,7 +232,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to target xid""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -293,7 +293,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery with target inclusive false""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -358,7 +358,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to target lsn""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -431,7 +431,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to target lsn""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -505,7 +505,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to latest from archive full+ptrack backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -553,7 +553,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery to latest from archive full+ptrack+ptrack backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -608,7 +608,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery in stream mode to latest from full + ptrack backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -663,7 +663,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -729,7 +729,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -794,7 +794,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery using tablespace-mapping option""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -913,7 +913,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """recovery using tablespace-mapping option and page backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -995,7 +995,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1045,7 +1045,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1094,7 +1094,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1143,7 +1143,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1198,7 +1198,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1247,7 +1247,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_zags_block_corrupt(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1303,7 +1303,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1325,7 +1325,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_zags_block_corrupt_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -1395,7 +1395,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.switch_wal_segment(node) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/retention_test.py b/tests/retention_test.py index 91fef215..34c02658 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -15,7 +15,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -72,7 +72,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -126,7 +126,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/show_test.py b/tests/show_test.py index 655c0b8e..eef89718 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -15,7 +15,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -43,7 +43,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -71,7 +71,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -117,7 +117,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -146,7 +146,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -177,7 +177,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/validate_test.py b/tests/validate_test.py index 6153a4a4..cdd960e7 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -21,7 +21,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -73,7 +73,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -213,7 +213,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -298,7 +298,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -421,7 +421,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -620,7 +620,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -804,7 +804,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -952,7 +952,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, try to restore backup with --no-validation option""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1038,7 +1038,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): corrupt file in FULL backup and run validate on instance, expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1113,7 +1113,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1175,7 +1175,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): def test_validate_corrupt_wal_2(self): """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1247,7 +1247,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1325,7 +1325,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1419,7 +1419,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1461,7 +1461,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1491,7 +1491,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node1 = self.make_simple_node( - base_dir="{0}/{1}/node1".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1506,7 +1506,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): backup_dir, 'node1', node1, options=["--stream"]) node2 = self.make_simple_node( - base_dir="{0}/{1}/node2".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node2')) node2.cleanup() node1.psql( @@ -1582,7 +1582,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1708,7 +1708,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1820,7 +1820,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2170,7 +2170,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2394,7 +2394,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), # initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -2472,7 +2472,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2614,7 +2614,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2735,7 +2735,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2923,7 +2923,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -3071,7 +3071,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): """ PGPRO-2096 """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} From 361d408c19cebb7d24e826e11cc8a86095dd6a78 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 13:00:57 +0300 Subject: [PATCH 42/59] tests: support new behaviour of node.cleanup() in testgres --- tests/restore_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index 92c85a79..9c2604d1 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -5,6 +5,7 @@ import subprocess from datetime import datetime import sys import time +import shutil module_name = 'restore' @@ -840,7 +841,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) # 2 - Try to restore to existing tablespace directory - node.cleanup() + shutil.rmtree(node.data_dir, ignore_errors=True) try: self.restore_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen From c7dee40d134c290f9ece90297327f31f2fd0012a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 13:04:16 +0300 Subject: [PATCH 43/59] tests: windows support for test_restore_with_tablespace_mapping_1 --- tests/restore_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index 9c2604d1..db69538c 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -833,10 +833,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual( + self.assertIn( + 'ERROR: restore destination is not empty: "{0}"'.format(node.data_dir), e.message, - 'ERROR: restore destination is not empty: "{0}"\n'.format( - node.data_dir), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 8f9ca4ae9f445ec824139300f116d5c447dd5a2c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 13:18:19 +0300 Subject: [PATCH 44/59] tests: windows support for test_restore_with_tablespace_mapping_1 --- tests/restore_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index db69538c..35933621 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -840,7 +840,10 @@ class RestoreTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) # 2 - Try to restore to existing tablespace directory - shutil.rmtree(node.data_dir, ignore_errors=True) + tblspc_path_tmp = os.path.join(node.base_dir, "tblspc_tmp") + os.rename(tblspc_path, tblspc_path_tmp) + node.cleanup() + os.rename(tblspc_path_tmp, tblspc_path) try: self.restore_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen @@ -850,10 +853,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "not empty.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual( + self.assertIn( + 'ERROR: restore tablespace destination is not empty:', e.message, - 'ERROR: restore tablespace destination ' - 'is not empty: "{0}"\n'.format(tblspc_path), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 68ac0b43e26f443abd91c149e5d76fb1a9fd4238 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 13:32:46 +0300 Subject: [PATCH 45/59] README: Windows support in beta --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c2267d0..a356b81b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `pg_probackup` currently has the following limitations: * Creating backups from a remote server is currently not supported. * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. -* Microsoft Windows operating system is not supported. +* Microsoft Windows operating system support is in beta stage. * Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately. ## Installation and Setup @@ -71,13 +71,28 @@ yum install pg_probackup-{11,10,9.6,9.5} yumdownloader --source pg_probackup-{11,10,9.6,9.5} ``` +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). + +## Building from source +### Linux + To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: ```shell make USE_PGXS=1 PG_CONFIG= top_srcdir= ``` +### Windows -Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). +Currently pg_probackup can be build using only MSVC 2013. +Build PostgreSQL using [pgwininstall](https://github.com/postgrespro/pgwininstall) or [PostgreSQL instruction](https://www.postgresql.org/docs/10/install-windows-full.html) with MSVC 2013. +If zlib support is needed, src/tools/msvc/config.pl must contain path to directory with compiled zlib. [Example](https://gist.githubusercontent.com/gsmol/80989f976ce9584824ae3b1bfb00bd87/raw/240032950d4ac4801a79625dd00c8f5d4ed1180c/gistfile1.txt) + +```shell +CALL "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64 +SET PATH=%PATH%;C:\Perl64\bin +SET PATH=%PATH%;C:\msys64\usr\bin +gen_probackup_project.pl C:\path_to_postgresql_source_tree +``` ## Documentation From f6c1966415df3bf81e5dee06b7c3445d9e6ea9fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 17:19:23 +0300 Subject: [PATCH 46/59] tests: windows support, eschew using node.restart() --- tests/auth_test.py | 3 ++- tests/ptrack.py | 6 ++++-- tests/replica.py | 3 ++- tests/restore_test.py | 1 - tests/validate_test.py | 3 ++- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/auth_test.py b/tests/auth_test.py index cb13e607..153044e1 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -126,7 +126,8 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase): "test1", "create table t1 as select generate_series(0,100)") node.append_conf("postgresql.auto.conf", "ptrack_enable = 'on'") - node.restart() + node.stop() + node.slow_start() try: self.backup_node( diff --git a/tests/ptrack.py b/tests/ptrack.py index 376e36a9..9f52a466 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -89,11 +89,13 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # DISABLE PTRACK node.safe_psql('postgres', "alter system set ptrack_enable to off") - node.restart() + node.stop() + node.slow_start() # ENABLE PTRACK node.safe_psql('postgres', "alter system set ptrack_enable to on") - node.restart() + node.stop() + node.slow_start() # PTRACK BACKUP try: diff --git a/tests/replica.py b/tests/replica.py index d9eabcbb..4eb4b1e2 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -403,7 +403,8 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): replica.append_conf( 'recovery.conf', "recovery_min_apply_delay = '300s'") - replica.restart() + replica.stop() + replica.slow_start() master.pgbench_init(scale=10) diff --git a/tests/restore_test.py b/tests/restore_test.py index 35933621..e130894a 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -5,7 +5,6 @@ import subprocess from datetime import datetime import sys import time -import shutil module_name = 'restore' diff --git a/tests/validate_test.py b/tests/validate_test.py index cdd960e7..1fc0c16e 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1528,7 +1528,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): node2.append_conf( 'postgresql.auto.conf', 'archive_mode = on') - node2.restart() + node2.stop() + node2.slow_start() timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] From 7d53afb1c749cdedbff7ac3b086acb24ec57e29b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 22:40:23 +0300 Subject: [PATCH 47/59] tests: windows support for validate --- tests/backup_test.py | 2 +- tests/merge.py | 6 ++- tests/validate_test.py | 94 +++++++++++++++++++++++++++--------------- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 24cd7b36..e3fdac15 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -484,7 +484,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): list = [] for root, dirs, files in os.walk(os.path.join( - backup_dir, 'backups/node', backup_id_1)): + backup_dir, 'backups', 'node', backup_id_1)): for file in files: if file == relfilenode: path = os.path.join(root, file) diff --git a/tests/merge.py b/tests/merge.py index dc5e3949..8d190811 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1164,7 +1164,8 @@ class MergeTest(ProbackupTest, unittest.TestCase): # read block from future # block_size + backup_header = 8200 file = os.path.join( - backup_dir, 'backups/node', backup_id_2, 'database', new_path) + backup_dir, 'backups', 'node', + backup_id_2, 'database', new_path) with open(file, 'rb') as f: f.seek(8200) block_1 = f.read(8200) @@ -1172,7 +1173,8 @@ class MergeTest(ProbackupTest, unittest.TestCase): # write block from future file = os.path.join( - backup_dir, 'backups/node', backup_id, 'database', old_path) + backup_dir, 'backups', 'node', + backup_id, 'database', old_path) with open(file, 'r+b') as f: f.seek(8200) f.write(block_1) diff --git a/tests/validate_test.py b/tests/validate_test.py index 1fc0c16e..e40fb389 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -249,7 +249,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Corrupt some file file = os.path.join( - backup_dir, 'backups/node', backup_id_2, 'database', file_path) + backup_dir, 'backups', 'node', + backup_id_2, 'database', file_path) with open(file, "r+b", 0) as f: f.seek(42) f.write(b"blah") @@ -342,7 +343,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Corrupt some file in FULL backup file_full = os.path.join( - backup_dir, 'backups/node', + backup_dir, 'backups', 'node', backup_id_1, 'database', file_path_t_heap) with open(file_full, "rb+", 0) as f: f.seek(84) @@ -352,7 +353,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Corrupt some file in PAGE1 backup file_page1 = os.path.join( - backup_dir, 'backups/node', + backup_dir, 'backups', 'node', backup_id_2, 'database', file_path_t_heap_1) with open(file_page1, "rb+", 0) as f: f.seek(42) @@ -379,8 +380,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertTrue( 'INFO: Validating backup {0}'.format( backup_id_1) in e.message and - 'WARNING: Invalid CRC of backup file "{0}"'.format( - file_full) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format( backup_id_1) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -499,7 +499,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Corrupt some file in PAGE2 and PAGE5 backups file_page1 = os.path.join( - backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + backup_dir, 'backups', 'node', backup_id_3, 'database', file_page_2) with open(file_page1, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -507,7 +507,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): f.close file_page4 = os.path.join( - backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + backup_dir, 'backups', 'node', backup_id_6, 'database', file_page_5) with open(file_page4, "rb+", 0) as f: f.seek(42) f.write(b"blah") @@ -547,8 +547,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertTrue( 'INFO: Validating backup {0}'.format( backup_id_3) in e.message and - 'WARNING: Invalid CRC of backup file "{0}"'.format( - file_page1) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format( backup_id_3) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -698,7 +697,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Corrupt some file in PAGE2 and PAGE5 backups file_page1 = os.path.join( - backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + backup_dir, 'backups', 'node', + backup_id_3, 'database', file_page_2) with open(file_page1, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -706,7 +706,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): f.close file_page4 = os.path.join( - backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + backup_dir, 'backups', 'node', + backup_id_6, 'database', file_page_5) with open(file_page4, "rb+", 0) as f: f.seek(42) f.write(b"blah") @@ -747,8 +748,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertTrue( 'INFO: Validating backup {0}'.format( backup_id_3) in e.message and - 'WARNING: Invalid CRC of backup file "{0}"'.format( - file_page1) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format( backup_id_3) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -851,7 +851,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Corrupt some file in FULL backup file_full = os.path.join( - backup_dir, 'backups/node', backup_id_2, + backup_dir, 'backups', 'node', backup_id_2, 'database', file_path_t_heap1) with open(file_full, "rb+", 0) as f: f.seek(84) @@ -906,8 +906,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertTrue( 'INFO: Validating backup {0}'.format( backup_id_2) in e.message and - 'WARNING: Invalid CRC of backup file "{0}"'.format( - file_full) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format( backup_id_2) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -964,7 +963,9 @@ class ValidateTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") file_path_t_heap = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").rstrip() @@ -973,14 +974,18 @@ class ValidateTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") # PAGE1 backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') # PAGE2 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') # FULL1 @@ -989,11 +994,15 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # PAGE3 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + file_full = os.path.join( + backup_dir, 'backups', 'node', + backup_id_1, 'database', file_path_t_heap) with open(file_full, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -1009,7 +1018,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertTrue( 'INFO: Validating backup {0}'.format(backup_id_1) in e.message and "INFO: Validate backups of the instance 'node'" in e.message - and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -1050,7 +1059,10 @@ class ValidateTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").rstrip() @@ -1059,27 +1071,40 @@ class ValidateTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + # PAGE1 - backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # PAGE2 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") - backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # FULL1 - backup_id_4 = self.backup_node(backup_dir, 'node', node) + backup_id_4 = self.backup_node( + backup_dir, 'node', node) # PAGE3 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + file_full = os.path.join( + backup_dir, 'backups', 'node', + backup_id_1, 'database', file_path_t_heap) with open(file_full, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -1089,13 +1114,16 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Validate Instance try: self.validate_pb(backup_dir, 'node') - self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( 'INFO: Validating backup {0}'.format(backup_id_1) in e.message and "INFO: Validate backups of the instance 'node'" in e.message - and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) From 723c2b01a26897a4d0f16ff8066be24e75fe2230 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 28 Dec 2018 18:46:12 +0300 Subject: [PATCH 48/59] Bug fix: Consider target_lsn during validate WAL --- src/parsexlog.c | 24 ++++++++++++++++++------ src/pg_probackup.c | 8 ++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index a1ca14da..b645ca70 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -519,6 +519,7 @@ validate_wal(pgBackup *backup, const char *archivedir, TimestampTz last_time = 0; char last_timestamp[100], target_timestamp[100]; + XLogRecPtr last_lsn = InvalidXLogRecPtr; bool all_wal = false; char backup_xlog_path[MAXPGPATH]; @@ -585,6 +586,7 @@ validate_wal(pgBackup *backup, const char *archivedir, /* We can restore at least up to the backup end */ time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); last_xid = backup->recovery_xid; + last_lsn = backup->stop_lsn; if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) || (target_time != 0 && backup->recovery_time >= target_time) @@ -608,6 +610,7 @@ validate_wal(pgBackup *backup, const char *archivedir, timestamp_record = getRecordTimestamp(xlogreader, &last_time); if (XLogRecGetXid(xlogreader) != InvalidTransactionId) last_xid = XLogRecGetXid(xlogreader); + last_lsn = xlogreader->ReadRecPtr; /* Check target xid */ if (TransactionIdIsValid(target_xid) && target_xid == last_xid) @@ -616,12 +619,19 @@ validate_wal(pgBackup *backup, const char *archivedir, break; } /* Check target time */ - else if (target_time != 0 && timestamp_record && timestamptz_to_time_t(last_time) >= target_time) + else if (target_time != 0 && timestamp_record && + timestamptz_to_time_t(last_time) >= target_time) { all_wal = true; break; } - /* If there are no target xid and target time */ + /* Check target lsn */ + else if (XRecOffIsValid(target_xid) && last_lsn >= target_lsn) + { + all_wal = true; + break; + } + /* If there are no target xid, target time and target lsn */ else if (!TransactionIdIsValid(target_xid) && target_time == 0 && xlogreader->ReadRecPtr == backup->stop_lsn) { @@ -638,15 +648,17 @@ validate_wal(pgBackup *backup, const char *archivedir, /* There are all needed WAL records */ if (all_wal) - elog(INFO, "backup validation completed successfully on time %s and xid " XID_FMT, - last_timestamp, last_xid); + elog(INFO, "backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", + last_timestamp, last_xid, + (uint32) (last_lsn >> 32), (uint32) last_lsn); /* Some needed WAL records are absent */ else { PrintXLogCorruptionMsg(&private, WARNING); - elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, - last_timestamp, last_xid); + elog(WARNING, "recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", + last_timestamp, last_xid, + (uint32) (last_lsn >> 32), (uint32) last_lsn); if (target_time > 0) time2iso(target_timestamp, lengthof(target_timestamp), diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 10478970..2cca5a2f 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -69,10 +69,10 @@ bool smooth_checkpoint; bool is_remote_backup = false; /* restore options */ -static char *target_time; -static char *target_xid; -static char *target_lsn; -static char *target_inclusive; +static char *target_time = NULL; +static char *target_xid = NULL; +static char *target_lsn = NULL; +static char *target_inclusive = NULL; static TimeLineID target_tli; static bool target_immediate; static char *target_name = NULL; From c69d4ed7f81e9ad0e904ece9a66845829bc27d1b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 28 Dec 2018 19:24:55 +0300 Subject: [PATCH 49/59] tests: validate fix (windows support) --- tests/validate_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index e40fb389..3c1aee59 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -56,7 +56,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): with open(log_file_path) as f: self.assertTrue( - 'LOG: File: {0} blknum 1, empty page'.format(file) in f.read(), + '{0} blknum 1, empty page'.format(file_path) in f.read(), 'Failed to detect nullified block') self.validate_pb(backup_dir) @@ -3134,6 +3134,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.run_binary( [ pg_resetxlog_path, + '-D', os.path.join(backup_dir, 'backups', 'node', backup_id, 'database'), '-o 42', '-f' From 93cedf1747595b0713b21081243a91dd439be6d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 29 Dec 2018 12:42:59 +0300 Subject: [PATCH 50/59] tests: fix test_validate_wal_unreal_values() and test_validate_wal_lost_segment_1() --- tests/validate_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index 3c1aee59..f2bf52fe 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -117,9 +117,9 @@ class ValidateTest(ProbackupTest, unittest.TestCase): "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual( + self.assertIn( + 'ERROR: Backup satisfying target options is not found', e.message, - 'ERROR: Backup satisfying target options is not found.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1309,8 +1309,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WAL segment \"{0}\" is absent".format( - file) in e.message and + "is absent" in e.message and "WARNING: There are not enough WAL records to consistenly " "restore backup {0}".format(backup_id) in e.message and "WARNING: Backup {0} WAL segments are corrupted".format( From 687b069f091986cf9867343ec5a4bf11db918b08 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 29 Dec 2018 17:13:34 +0300 Subject: [PATCH 51/59] Bug fix: Fix for 723c2b01a --- src/parsexlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index b645ca70..44c87ff8 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -626,14 +626,14 @@ validate_wal(pgBackup *backup, const char *archivedir, break; } /* Check target lsn */ - else if (XRecOffIsValid(target_xid) && last_lsn >= target_lsn) + else if (XRecOffIsValid(target_lsn) && last_lsn >= target_lsn) { all_wal = true; break; } /* If there are no target xid, target time and target lsn */ else if (!TransactionIdIsValid(target_xid) && target_time == 0 && - xlogreader->ReadRecPtr == backup->stop_lsn) + !XRecOffIsValid(target_lsn)) { all_wal = true; /* We don't stop here. We want to get last_xid and last_time */ From d8553c06afff82a39ab217c4ca34b6a760934c1d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 9 Jan 2019 15:28:30 +0300 Subject: [PATCH 52/59] Version 2.0.26 - Bugfix: fixed undefined behaviour in case of undefined compression algorithm - Bugfix: correctly calculate backup size after MERGE - Bugfix: fixed MERGE of compressed and uncompressed backups - Bugfix: remove unnecessary remaining files after MERGE - Bugfix: consider target LSN (--lsn parameter) during validating WAL - Improvement: check backup program_version during VALIDATE, do not support forward compatibility - Improvement: improve Windows support - Improvement: improve support of tablespaces within PGDATA --- src/pg_probackup.c | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 2cca5a2f..e5057686 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -19,7 +19,7 @@ #include "utils/thread.h" #include -const char *PROGRAM_VERSION = "2.0.25"; +const char *PROGRAM_VERSION = "2.0.26"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 1f2b8a4e..17afa4fe 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.25 \ No newline at end of file +pg_probackup 2.0.26 \ No newline at end of file From 1fff8a2d3d238a089c13e6a4343bf2f36601d22e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 9 Jan 2019 16:58:37 +0300 Subject: [PATCH 53/59] tests: pass replica=True to replica for slow_start() --- tests/replica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replica.py b/tests/replica.py index 4eb4b1e2..6d6fb33f 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -404,7 +404,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): 'recovery.conf', "recovery_min_apply_delay = '300s'") replica.stop() - replica.slow_start() + replica.slow_start(replica=True) master.pgbench_init(scale=10) From dcdd21300b8306c740dfaa5ffe201592bc5c1c8a Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 9 Jan 2019 18:25:52 +0300 Subject: [PATCH 54/59] tests: do not call exit(1) --- tests/replica.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index 6d6fb33f..9ab49a6e 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -3,7 +3,6 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta import subprocess -from sys import exit import time @@ -52,7 +51,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # Check data correctness on replica replica.slow_start(replica=True) - exit(1) after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) From 069b9c10601c31df568c6e9e3aab900212b1f1b8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 10 Jan 2019 18:12:00 +0300 Subject: [PATCH 55/59] tests: python 3 support --- tests/archive.py | 4 ++-- tests/helpers/ptrack_helpers.py | 26 +++++++++++++------------- tests/ptrack_clean.py | 2 +- tests/validate_test.py | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 1d0d7af5..0881a050 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -742,7 +742,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', '-D', os.path.join(backup_dir, 'wal', 'node') - ], async=True) + ], asynchronous=True) if pg_receivexlog.returncode: self.assertFalse( @@ -816,7 +816,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') - ], async=True) + ], asynchronous=True) if pg_receivexlog.returncode: self.assertFalse( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 17fc4f23..b1999864 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -124,7 +124,11 @@ def slow_start(self, replica=False): # raise_operational_error=False) self.start() - if not replica: + if replica: + self.poll_query_until( + 'postgres', + 'SELECT pg_is_in_recovery()') + else: while True: try: self.poll_query_until( @@ -133,10 +137,6 @@ def slow_start(self, replica=False): break except Exception as e: continue - else: - self.poll_query_until( - 'postgres', - 'SELECT pg_is_in_recovery()') # while True: # try: @@ -551,7 +551,7 @@ class ProbackupTest(object): ) ) - def run_pb(self, command, async=False, gdb=False, old_binary=False): + def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False): if not self.probackup_old_path and old_binary: print('PGPROBACKUPBIN_OLD is not set') exit(1) @@ -567,7 +567,7 @@ class ProbackupTest(object): print(self.cmd) if gdb: return GDBobj([binary_path] + command, self.verbose) - if async: + if asynchronous: return subprocess.Popen( self.cmd, stdout=subprocess.PIPE, @@ -590,11 +590,11 @@ class ProbackupTest(object): except subprocess.CalledProcessError as e: raise ProbackupException(e.output.decode('utf-8'), self.cmd) - def run_binary(self, command, async=False): + def run_binary(self, command, asynchronous=False): if self.verbose: print([' '.join(map(str, command))]) try: - if async: + if asynchronous: return subprocess.Popen( command, stdin=subprocess.PIPE, @@ -649,7 +649,7 @@ class ProbackupTest(object): def backup_node( self, backup_dir, instance, node, data_dir=False, - backup_type='full', options=[], async=False, gdb=False, + backup_type='full', options=[], asynchronous=False, gdb=False, old_binary=False ): if not node and not data_dir: @@ -673,10 +673,10 @@ class ProbackupTest(object): if backup_type: cmd_list += ['-b', backup_type] - return self.run_pb(cmd_list + options, async, gdb, old_binary) + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) def merge_backup( - self, backup_dir, instance, backup_id, async=False, + self, backup_dir, instance, backup_id, asynchronous=False, gdb=False, old_binary=False, options=[]): cmd_list = [ 'merge', @@ -685,7 +685,7 @@ class ProbackupTest(object): '-i', backup_id ] - return self.run_pb(cmd_list + options, async, gdb, old_binary) + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) def restore_node( self, backup_dir, instance, node=False, diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 01d364e9..3eeba1b7 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -149,7 +149,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start() + replica.slow_start(replica=True) # Create table and indexes master.safe_psql( diff --git a/tests/validate_test.py b/tests/validate_test.py index f2bf52fe..c6d32906 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -2453,7 +2453,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node( backup_dir, 'node', node, backup_type="full", - options=["-j", "4"], async=False, gdb=False) + options=["-j", "4"], asynchronous=False, gdb=False) node.stop() node.cleanup() @@ -3138,7 +3138,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '-o 42', '-f' ], - async=False) + asynchronous=False) md5_after = hashlib.md5( open(pg_control_path, 'rb').read()).hexdigest() From 1b2ead72df3fe41eae45c0435344a3819389f172 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 10 Jan 2019 18:17:03 +0300 Subject: [PATCH 56/59] tests: python3 minor fix --- tests/helpers/ptrack_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b1999864..286255c9 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1229,7 +1229,7 @@ class ProbackupTest(object): database='postgres', host='127.0.0.1', port=port, - async=True + async_=True ) def wait(self, connection): From 01770e7e37fdba9bcf491c822d61b7bcabd64fda Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Jan 2019 17:21:06 +0300 Subject: [PATCH 57/59] tests: slow_start for replicas --- tests/ptrack_cluster.py | 4 ++-- tests/ptrack_empty.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index 270cb1b4..7fe312b3 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -198,7 +198,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start() + replica.slow_start(replica=True) # Create table and indexes master.safe_psql( @@ -298,7 +298,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start() + replica.slow_start(replica=True) # Create table and indexes master.safe_psql( diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index 578c8a73..35583ff1 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -116,7 +116,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start() + replica.slow_start(replica=True) # Create table master.safe_psql( From 8c7e1aa85dcf2cd325c636ed8e6727cb8b2dbcc5 Mon Sep 17 00:00:00 2001 From: Ilya Skvortsov Date: Fri, 11 Jan 2019 17:44:39 +0300 Subject: [PATCH 58/59] try to continue MERGE after fail on backup deleting --- tests/merge.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 8d190811..27fc2dc6 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1198,6 +1198,93 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + def test_continue_failed_merge_2(self): + """ + Check that failed MERGE on delete can be continued + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id = self.show_pb(backup_dir, "node")[2]["id"] + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + + gdb.set_breakpoint('pgFileDelete') + gdb.run_until_break() + + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + gdb._execute('signal SIGKILL') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + backup_id_deleted = self.show_pb(backup_dir, "node")[1]["id"] + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Backup {0} has status: DELETING".format( + backup_id_deleted) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From 5bcea32f31284233d380990b652f2a73f2aee3cc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Jan 2019 17:54:50 +0300 Subject: [PATCH 59/59] tests: comment minor fix --- tests/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index 27fc2dc6..4a0cb31c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1200,7 +1200,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): def test_continue_failed_merge_2(self): """ - Check that failed MERGE on delete can be continued + Check that failed MERGE on delete can`t be continued """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')