From 55238d572fb4d463ed3336ffa083df5260bdd2ee Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Mon, 20 Jun 2022 13:44:42 +0300 Subject: [PATCH 01/55] [PBCKP-120] skip partitioned indexes for checkdb --amcheck Tags: pg_probackup --- src/checkdb.c | 10 +++++++--- tests/checkdb.py | 9 +++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/checkdb.c b/src/checkdb.c index 177fc3cc..1133a7b5 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -461,7 +461,9 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " "ORDER BY nmspc.nspname DESC", 0, NULL); } @@ -473,8 +475,10 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' AND " - "(cls.reltablespace IN " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " + "AND (cls.reltablespace IN " "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " "OR cls.reltablespace = 0) " "ORDER BY nmspc.nspname DESC", diff --git a/tests/checkdb.py b/tests/checkdb.py index 9b7adcd7..4608366b 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -36,6 +36,15 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", "create index on t_heap(id)") + + node.safe_psql( + "postgres", + "create table idxpart (a int) " + "partition by range (a)") + + node.safe_psql( + "postgres", + "create index on idxpart(a)") try: node.safe_psql( From e918bae0dc7c5b4f8b5bb6748a47485c526e803f Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 26 Aug 2022 20:00:58 +0300 Subject: [PATCH 02/55] Version 2.5.8 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8aad0a7c..802cbb5c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -338,7 +338,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.7" +#define PROGRAM_VERSION "2.5.8" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index af186a98..4de28890 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.7 +pg_probackup 2.5.8 From 3d7b9f0e807741a546f3d95eb48106e120a89bf7 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Sun, 28 Aug 2022 00:39:33 +0300 Subject: [PATCH 03/55] =?UTF-8?q?[PBCKP-258]=20fix=20multiple=20permission?= =?UTF-8?q?=20tests=20(revert=20a3ac7d5e7a8d6ebeafd=E2=80=A6=20(#527)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 2 ++ tests/backup.py | 58 +++++++++++++++--------------------------------- tests/checkdb.py | 31 +++++++++++++------------- tests/restore.py | 14 +++++------- 4 files changed, 41 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26b2bc4e..a7dae2ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ env: # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=compression # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=locking @@ -52,6 +53,7 @@ env: jobs: allow_failures: - if: env(PG_BRANCH) = master + - if: env(PG_BRANCH) = REL_15_STABLE - if: env(PG_BRANCH) = REL9_5_STABLE # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) diff --git a/tests/backup.py b/tests/backup.py index 7d02f5b3..c5235120 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1889,8 +1889,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -1928,9 +1927,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -1967,9 +1964,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") if self.ptrack: node.safe_psql( @@ -1982,10 +1977,10 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION ptrack.ptrack_init_lsn() TO backup;") if ProbackupTest.enterprise: - node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # FULL backup self.backup_node( @@ -2245,7 +2240,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): if self.get_version(node) < 90600: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2256,14 +2250,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2275,14 +2266,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2294,9 +2282,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # enable STREAM backup node.safe_psql( @@ -3067,9 +3053,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -3091,15 +3075,14 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") - + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") + sleep(2) replica.promote() @@ -3177,8 +3160,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -3201,9 +3183,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -3225,15 +3205,13 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") replica.promote() diff --git a/tests/checkdb.py b/tests/checkdb.py index 2df946cf..bcda0fb2 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -666,8 +666,8 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' # amcheck-next function - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') # amcheck-next function + # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -696,9 +696,8 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' # 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + # PG 10 elif self.get_version(node) > 100000 and self.get_version(node) < 110000: node.safe_psql( @@ -726,10 +725,8 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;') + if ProbackupTest.enterprise: # amcheck-1.1 node.safe_psql( @@ -768,9 +765,8 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + # checkunique parameter if ProbackupTest.enterprise: if (self.get_version(node) >= 111300 and self.get_version(node) < 120000 @@ -807,15 +803,20 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + # checkunique parameter if ProbackupTest.enterprise: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") + if ProbackupTest.enterprise: + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') + # checkdb try: self.checkdb_node( diff --git a/tests/restore.py b/tests/restore.py index 37f13357..b619078d 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3230,8 +3230,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -3269,9 +3268,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -3307,9 +3304,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") if self.ptrack: # TODO why backup works without these grants ? @@ -3326,7 +3321,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # FULL backup without database_map backup_id = self.backup_node( From 7d3e7f864c0e373cc766df62ecef420dfef87bad Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 29 Aug 2022 05:04:05 +0300 Subject: [PATCH 04/55] [PBCKP-129] fix tests.replica.ReplicaTest.test_replica_stop_lsn_null_offset_next_record --- tests/replica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replica.py b/tests/replica.py index acf655aa..24dbaa39 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -806,7 +806,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): log_content) self.assertIn( - 'LOG: stop_lsn: 0/4000000', + 'INFO: stop_lsn: 0/4000000', log_content) self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') From 95471acc75434834881a260f6e5299c1822bfcad Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Tue, 30 Aug 2022 12:42:47 +0300 Subject: [PATCH 05/55] [PBCKP-257] fix time_consuming.TimeConsumingTests.test_pbckp150 (#525) --- tests/time_consuming.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/time_consuming.py b/tests/time_consuming.py index 396ab716..c778b9bc 100644 --- a/tests/time_consuming.py +++ b/tests/time_consuming.py @@ -15,22 +15,28 @@ class TimeConsumingTests(ProbackupTest, unittest.TestCase): run pgbench, vacuum VERBOSE FULL and ptrack backups in parallel """ # init node + if self.pg_config_version < self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL >= 11 for this test') + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums']) - node.append_conf('postgresql.conf', - """ - max_connections = 100 - wal_keep_size = 16000 - ptrack.map_size = 1 - shared_preload_libraries='ptrack' - log_statement = 'none' - fsync = off - log_checkpoints = on - autovacuum = off - """) + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={ + 'max_connections': 100, + 'log_statement': 'none', + 'log_checkpoints': 'on', + 'autovacuum': 'off', + 'ptrack.map_size': 1}) + + if node.major_version >= 13: + self.set_auto_conf(node, {'wal_keep_size': '16000MB'}) + else: + self.set_auto_conf(node, {'wal_keep_segments': '1000'}) # init probackup and add an instance backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From 1bb07627295ba44ed059d1d700db01d3a9799915 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 1 Sep 2022 14:38:17 +0300 Subject: [PATCH 06/55] =?UTF-8?q?[PBCKP-259]=20fix=20for=20'ERROR:=20Canno?= =?UTF-8?q?t=20create=20directory=20for=20older=20backup'=E2=80=A6=20(#526?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [PBCKP-259] fix for 'ERROR: Cannot create directory for older backup', rewrite --start_time implementation * rewritten 5f2283c8deac88ea49ea6223a3aa72e2cf462eb5 * fixes for several tests * disabled tests.merge.MergeTest.test_merge_backup_from_future and tests.restore.RestoreTest.test_restore_backup_from_future as incorrect for now Co-authored-by: d.lepikhova --- src/backup.c | 60 ++++- src/catalog.c | 98 ++++----- src/pg_probackup.c | 8 +- src/pg_probackup.h | 7 +- tests/backup.py | 374 ++++++++------------------------ tests/helpers/ptrack_helpers.py | 25 ++- tests/merge.py | 6 +- tests/restore.py | 6 +- 8 files changed, 222 insertions(+), 362 deletions(-) diff --git a/src/backup.c b/src/backup.c index 9a451e72..03ff7b72 100644 --- a/src/backup.c +++ b/src/backup.c @@ -692,6 +692,8 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) /* * Entry point of pg_probackup BACKUP subcommand. + * + * if start_time == INVALID_BACKUP_ID then we can generate backup_id */ int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, @@ -699,8 +701,13 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; + time_t latest_backup_id = INVALID_BACKUP_ID; char pretty_bytes[20]; + if (!instance_config.pgdata) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); + /* Initialize PGInfonode */ pgNodeInit(&nodeInfo); @@ -709,12 +716,55 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, (pg_strcasecmp(instance_config.external_dir_str, "none") != 0)) current.external_dir_str = instance_config.external_dir_str; - /* Create backup directory and BACKUP_CONTROL_FILE */ - pgBackupCreateDir(¤t, instanceState, start_time); + /* Find latest backup_id */ + { + parray *backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - if (!instance_config.pgdata) - elog(ERROR, "required parameter not specified: PGDATA " - "(-D, --pgdata)"); + if (parray_num(backup_list) > 0) + latest_backup_id = ((pgBackup *)parray_get(backup_list, 0))->backup_id; + + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + } + + /* Try to pick backup_id and create backup directory with BACKUP_CONTROL_FILE */ + if (start_time != INVALID_BACKUP_ID) + { + /* If user already choosed backup_id for us, then try to use it. */ + if (start_time <= latest_backup_id) + /* don't care about freeing base36enc_dup memory, we exit anyway */ + elog(ERROR, "Can't assign backup_id from requested start_time (%s), " + "this time must be later that backup %s", + base36enc_dup(start_time), base36enc_dup(latest_backup_id)); + + current.backup_id = start_time; + pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); + } + else + { + /* We can generate our own unique backup_id + * Sometimes (when we try to backup twice in one second) + * backup_id will be duplicated -> try more times. + */ + int attempts = 10; + + if (time(NULL) < latest_backup_id) + elog(ERROR, "Can't assign backup_id, there is already a backup in future (%s)", + base36enc(latest_backup_id)); + + do + { + current.backup_id = time(NULL); + pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); + if (current.backup_id == INVALID_BACKUP_ID) + sleep(1); + } + while (current.backup_id == INVALID_BACKUP_ID && attempts-- > 0); + } + + /* If creation of backup dir was unsuccessful, there will be WARNINGS in logs already */ + if (current.backup_id == INVALID_BACKUP_ID) + elog(ERROR, "Can't create backup directory"); /* Update backup status and other metainfo. */ current.status = BACKUP_STATUS_RUNNING; diff --git a/src/catalog.c b/src/catalog.c index c118e954..47513096 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -23,7 +23,7 @@ static pgBackup* get_closest_backup(timelineInfo *tlinfo); static pgBackup* get_oldest_backup(timelineInfo *tlinfo); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); -static void create_backup_dir(pgBackup *backup, const char *backup_instance_path); +static int create_backup_dir(pgBackup *backup, const char *backup_instance_path); static bool backup_lock_exit_hook_registered = false; static parray *locks = NULL; @@ -969,6 +969,7 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id } else if (strcmp(base36enc(backup->start_time), data_ent->d_name) != 0) { + /* TODO there is no such guarantees */ elog(WARNING, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", base36enc(backup->start_time), backup_conf_path); } @@ -1411,22 +1412,34 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, return NULL; } -/* Create backup directory in $BACKUP_PATH - * Note, that backup_id attribute is updated, - * so it is possible to get diffrent values in +/* + * Create backup directory in $BACKUP_PATH + * (with proposed backup->backup_id) + * and initialize this directory. + * If creation of directory fails, then + * backup_id will be cleared (set to INVALID_BACKUP_ID). + * It is possible to get diffrent values in * pgBackup.start_time and pgBackup.backup_id. * It may be ok or maybe not, so it's up to the caller * to fix it or let it be. */ void -pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time) +pgBackupInitDir(pgBackup *backup, const char *backup_instance_path) { - int i; - parray *subdirs = parray_new(); - parray * backups; - pgBackup *target_backup; + int i; + char temp[MAXPGPATH]; + parray *subdirs; + /* Try to create backup directory at first */ + if (create_backup_dir(backup, backup_instance_path) != 0) + { + /* Clear backup_id as indication of error */ + backup->backup_id = INVALID_BACKUP_ID; + return; + } + + subdirs = parray_new(); parray_append(subdirs, pg_strdup(DATABASE_DIR)); /* Add external dirs containers */ @@ -1438,7 +1451,6 @@ pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_t false); for (i = 0; i < parray_num(external_list); i++) { - char temp[MAXPGPATH]; /* Numeration of externaldirs starts with 1 */ makeExternalDirPathByNum(temp, EXTERNAL_DIR, i+1); parray_append(subdirs, pg_strdup(temp)); @@ -1446,30 +1458,6 @@ pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_t free_dir_list(external_list); } - /* Get list of all backups*/ - backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - if (parray_num(backups) > 0) - { - target_backup = (pgBackup *) parray_get(backups, 0); - if (start_time > target_backup->backup_id) - { - backup->backup_id = start_time; - create_backup_dir(backup, instanceState->instance_backup_subdir_path); - } - else - { - elog(ERROR, "Cannot create directory for older backup"); - } - } - else - { - backup->backup_id = start_time; - create_backup_dir(backup, instanceState->instance_backup_subdir_path); - } - - if (backup->backup_id == 0) - elog(ERROR, "Cannot create backup directory: %s", strerror(errno)); - backup->database_dir = pgut_malloc(MAXPGPATH); join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); @@ -1479,10 +1467,8 @@ pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_t /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) { - char path[MAXPGPATH]; - - join_path_components(path, backup->root_dir, parray_get(subdirs, i)); - fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); + join_path_components(temp, backup->root_dir, parray_get(subdirs, i)); + fio_mkdir(temp, DIR_PERMISSION, FIO_BACKUP_HOST); } free_dir_list(subdirs); @@ -1491,34 +1477,26 @@ pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_t /* * Create root directory for backup, * update pgBackup.root_dir if directory creation was a success + * Return values (same as dir_create_dir()): + * 0 - ok + * -1 - error (warning message already emitted) */ -void +int create_backup_dir(pgBackup *backup, const char *backup_instance_path) { - int attempts = 10; + int rc; + char path[MAXPGPATH]; - while (attempts--) - { - int rc; - char path[MAXPGPATH]; + join_path_components(path, backup_instance_path, base36enc(backup->backup_id)); - join_path_components(path, backup_instance_path, base36enc(backup->backup_id)); - - /* TODO: add wrapper for remote mode */ - rc = dir_create_dir(path, DIR_PERMISSION, true); - - if (rc == 0) - { - backup->root_dir = pgut_strdup(path); - return; - } - else - { - elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno)); - sleep(1); - } - } + /* TODO: add wrapper for remote mode */ + rc = dir_create_dir(path, DIR_PERMISSION, true); + if (rc == 0) + backup->root_dir = pgut_strdup(path); + else + elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno)); + return rc; } /* diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5867bd49..1f6b6313 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -78,7 +78,7 @@ pid_t my_pid = 0; __thread int my_thread_num = 1; bool progress = false; bool no_sync = false; -time_t start_time = 0; +time_t start_time = INVALID_BACKUP_ID; #if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; bool temp_slot = false; @@ -202,7 +202,6 @@ static ConfigOption cmd_options[] = { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, { 'b', 134, "no-color", &no_color, SOURCE_CMD_STRICT }, - { 'U', 241, "start-time", &start_time, SOURCE_CMD_STRICT }, /* backup options */ { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, @@ -217,6 +216,7 @@ static ConfigOption cmd_options[] = { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, { 's', 238, "note", &backup_note, SOURCE_CMD_STRICT }, + { 'U', 241, "start-time", &start_time, SOURCE_CMD_STRICT }, /* catchup options */ { 's', 239, "source-pgdata", &catchup_source_pgdata, SOURCE_CMD_STRICT }, { 's', 240, "destination-pgdata", &catchup_destination_pgdata, SOURCE_CMD_STRICT }, @@ -975,9 +975,7 @@ main(int argc, char *argv[]) case BACKUP_CMD: { current.stream = stream_wal; - if (start_time == 0) - start_time = current_time; - else + if (start_time != INVALID_BACKUP_ID) elog(WARNING, "Please do not use the --start-time option to start backup. " "This is a service option required to work with other extensions. " "We do not guarantee future support for this flag."); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 802cbb5c..1885a191 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -450,7 +450,10 @@ struct pgBackup { BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/ time_t backup_id; /* Identifier of the backup. - * Currently it's the same as start_time */ + * By default it's the same as start_time + * but can be increased if same backup_id + * already exists. It can be also set by + * start_time parameter */ BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ TimeLineID tli; /* timeline of start and stop backup lsns */ XLogRecPtr start_lsn; /* backup's starting transaction log location */ @@ -985,7 +988,7 @@ extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync); -extern void pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time); +extern void pgBackupInitDir(pgBackup *backup, const char *backup_instance_path); extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); extern void pgBackupFree(void *backup); diff --git a/tests/backup.py b/tests/backup.py index c5235120..0cba8fe7 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1,7 +1,7 @@ import unittest import os from time import sleep, time -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException import shutil from distutils.dir_util import copy_tree from testgres import ProcessType, QueryException @@ -313,7 +313,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -459,7 +459,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -600,7 +600,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -3402,10 +3402,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_start_time(self): - + """Test, that option --start-time allows to set backup_id and restore""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3418,138 +3419,81 @@ class BackupTest(ProbackupTest, unittest.TestCase): # FULL backup startTime = int(time()) self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore FULL backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_full'), + backup_id=base36enc(startTime)) + + #FULL backup with incorrect start time + try: + startTime = str(int(time()-100000)) + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + 'Expecting Error because start time for new backup must be newer ' + '\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + r"ERROR: Can't assign backup_id from requested start_time \(\w*\), this time must be later that backup \w*\n", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) # DELTA backup startTime = int(time()) self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=['--stream', '--start-time', str(startTime)]) + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore DELTA backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_delta'), + backup_id=base36enc(startTime)) # PAGE backup startTime = int(time()) self.backup_node( - backup_dir, 'node', node, backup_type="page", - options=['--stream', '--start-time', str(startTime)]) + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore PAGE backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_page'), + backup_id=base36enc(startTime)) - if self.ptrack and node.major_version > 11: + # PTRACK backup + if self.ptrack: node.safe_psql( - "postgres", - "create extension ptrack") + 'postgres', + 'create extension ptrack') - # PTRACK backup startTime = int(time()) self.backup_node( - backup_dir, 'node', node, backup_type="ptrack", - options=['--stream', '--start-time', str(startTime)]) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_start_time_incorrect_time(self): - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - startTime = int(time()) - #backup with correct start time - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--start-time', str(startTime)]) - #backups with incorrect start time - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=['--stream', '--start-time', str(startTime-10000)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=['--stream', '--start-time', str(startTime-10000)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="page", - options=['--stream', '--start-time', str(startTime-10000)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - if self.ptrack and node.major_version > 11: - node.safe_psql( - "postgres", - "create extension ptrack") - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="page", - options=['--stream', '--start-time', str(startTime-10000)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore PTRACK backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_ptrack'), + backup_id=base36enc(startTime)) # Clean after yourself self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_start_time_few_nodes(self): - + """Test, that we can synchronize backup_id's for different DBs""" fname = self.id().split('.')[3] node1 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node1'), + set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3561,6 +3505,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): node2 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node2'), + set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3571,200 +3516,61 @@ class BackupTest(ProbackupTest, unittest.TestCase): node2.slow_start() # FULL backup - startTime = int(time()) + startTime = str(int(time())) self.backup_node( - backup_dir1, 'node1', node1, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) + backup_dir1, 'node1', node1, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) self.backup_node( - backup_dir2, 'node2', node2, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - + backup_dir2, 'node2', node2, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) show_backup1 = self.show_pb(backup_dir1, 'node1')[0] show_backup2 = self.show_pb(backup_dir2, 'node2')[0] self.assertEqual(show_backup1['id'], show_backup2['id']) # DELTA backup - startTime = int(time()) + startTime = str(int(time())) self.backup_node( - backup_dir1, 'node1', node1, backup_type="delta", - options=['--stream', '--start-time', str(startTime)]) + backup_dir1, 'node1', node1, backup_type='delta', + options=['--stream', '--start-time={0}'.format(startTime)]) self.backup_node( - backup_dir2, 'node2', node2, backup_type="delta", - options=['--stream', '--start-time', str(startTime)]) + backup_dir2, 'node2', node2, backup_type='delta', + options=['--stream', '--start-time={0}'.format(startTime)]) show_backup1 = self.show_pb(backup_dir1, 'node1')[1] show_backup2 = self.show_pb(backup_dir2, 'node2')[1] self.assertEqual(show_backup1['id'], show_backup2['id']) # PAGE backup - startTime = int(time()) + startTime = str(int(time())) self.backup_node( - backup_dir1, 'node1', node1, backup_type="page", - options=['--stream', '--start-time', str(startTime)]) + backup_dir1, 'node1', node1, backup_type='page', + options=['--stream', '--start-time={0}'.format(startTime)]) self.backup_node( - backup_dir2, 'node2', node2, backup_type="page", - options=['--stream', '--start-time', str(startTime)]) + backup_dir2, 'node2', node2, backup_type='page', + options=['--stream', '--start-time={0}'.format(startTime)]) show_backup1 = self.show_pb(backup_dir1, 'node1')[2] show_backup2 = self.show_pb(backup_dir2, 'node2')[2] self.assertEqual(show_backup1['id'], show_backup2['id']) # PTRACK backup - startTime = int(time()) - if self.ptrack and node1.major_version > 11: + if self.ptrack: node1.safe_psql( - "postgres", - "create extension ptrack") - self.backup_node( - backup_dir1, 'node1', node1, backup_type="ptrack", - options=['--stream', '--start-time', str(startTime)]) - - if self.ptrack and node2.major_version > 11: + 'postgres', + 'create extension ptrack') node2.safe_psql( - "postgres", - "create extension ptrack") + 'postgres', + 'create extension ptrack') + + startTime = str(int(time())) self.backup_node( - backup_dir2, 'node2', node2, backup_type="ptrack", - options=['--stream', '--start-time', str(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[3] - show_backup2 = self.show_pb(backup_dir2, 'node2')[3] - self.assertEqual(show_backup1['id'], show_backup2['id']) + backup_dir1, 'node1', node1, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[3] + show_backup2 = self.show_pb(backup_dir2, 'node2')[3] + self.assertEqual(show_backup1['id'], show_backup2['id']) # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_start_time_few_nodes_incorrect_time(self): - - fname = self.id().split('.')[3] - node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir1 = os.path.join(self.tmp_path, module_name, fname, 'backup1') - self.init_pb(backup_dir1) - self.add_instance(backup_dir1, 'node1', node1) - self.set_archiving(backup_dir1, 'node1', node1) - node1.slow_start() - - node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2'), - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir2 = os.path.join(self.tmp_path, module_name, fname, 'backup2') - self.init_pb(backup_dir2) - self.add_instance(backup_dir2, 'node2', node2) - self.set_archiving(backup_dir2, 'node2', node2) - node2.slow_start() - - # FULL backup - startTime = int(time()) - self.backup_node( - backup_dir1, 'node1', node1, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type="full", - options=['--stream', '--start-time', str(startTime-10000)]) - - show_backup1 = self.show_pb(backup_dir1, 'node1')[0] - show_backup2 = self.show_pb(backup_dir2, 'node2')[0] - self.assertGreater(show_backup1['id'], show_backup2['id']) - - # DELTA backup - startTime = int(time()) - self.backup_node( - backup_dir1, 'node1', node1, backup_type="delta", - options=['--stream', '--start-time', str(startTime)]) - # make backup with start time definitelly earlier, than existing - try: - self.backup_node( - backup_dir2, 'node2', node2, backup_type="delta", - options=['--stream', '--start-time', str(10000)]) - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - show_backup1 = self.show_pb(backup_dir1, 'node1')[1] - show_backup2 = self.show_pb(backup_dir2, 'node2')[0] - self.assertGreater(show_backup1['id'], show_backup2['id']) - - # PAGE backup - startTime = int(time()) - self.backup_node( - backup_dir1, 'node1', node1, backup_type="page", - options=['--stream', '--start-time', str(startTime)]) - # make backup with start time definitelly earlier, than existing - try: - self.backup_node( - backup_dir2, 'node2', node2, backup_type="page", - options=['--stream', '--start-time', str(10000)]) - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - show_backup1 = self.show_pb(backup_dir1, 'node1')[2] - show_backup2 = self.show_pb(backup_dir2, 'node2')[0] - self.assertGreater(show_backup1['id'], show_backup2['id']) - - # PTRACK backup - startTime = int(time()) - if self.ptrack and node1.major_version > 11: - node1.safe_psql( - "postgres", - "create extension ptrack") - self.backup_node( - backup_dir1, 'node1', node1, backup_type="ptrack", - options=['--stream', '--start-time', str(startTime)]) - - if self.ptrack and node2.major_version > 11: - node2.safe_psql( - "postgres", - "create extension ptrack") - # make backup with start time definitelly earlier, than existing - try: - self.backup_node( - backup_dir2, 'node2', node2, backup_type="ptrack", - options=['--stream', '--start-time', str(10000)]) - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # FULL backup - startTime = int(time()) - self.backup_node( - backup_dir1, 'node1', node1, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - - show_backup1 = self.show_pb(backup_dir1, 'node1')[4] - show_backup2 = self.show_pb(backup_dir2, 'node2')[1] - self.assertEqual(show_backup1['id'], show_backup2['id']) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 418ef4e1..59eb12ae 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -110,6 +110,26 @@ def is_nls_enabled(): return b'enable-nls' in p.communicate()[0] +def base36enc(number): + """Converts an integer to a base36 string.""" + alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + base36 = '' + sign = '' + + if number < 0: + sign = '-' + number = -number + + if 0 <= number < len(alphabet): + return sign + alphabet[number] + + while number != 0: + number, i = divmod(number, len(alphabet)) + base36 = alphabet[i] + base36 + + return sign + base36 + + class ProbackupException(Exception): def __init__(self, message, cmd): self.message = message @@ -947,7 +967,7 @@ class ProbackupTest(object): backup_type='full', datname=False, options=[], asynchronous=False, gdb=False, old_binary=False, return_id=True, no_remote=False, - env=None, startTime=None + env=None ): if not node and not data_dir: print('You must provide ether node or data_dir for backup') @@ -980,9 +1000,6 @@ class ProbackupTest(object): if not old_binary: cmd_list += ['--no-sync'] - if startTime: - cmd_list += ['--start-time', startTime] - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env) def checkdb_node( diff --git a/tests/merge.py b/tests/merge.py index 5f092543..4c374bdf 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1965,7 +1965,11 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + # Skipped, because backups from the future are invalid. + # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" + # now (PBCKP-259). We can conduct such a test again when we + # untie 'backup_id' from 'start_time' + @unittest.skip("skip") def test_merge_backup_from_future(self): """ take FULL backup, table PAGE backup from future, diff --git a/tests/restore.py b/tests/restore.py index b619078d..ae1c7cbe 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1853,7 +1853,11 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + # Skipped, because backups from the future are invalid. + # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" + # now (PBCKP-259). We can conduct such a test again when we + # untie 'backup_id' from 'start_time' + @unittest.skip("skip") def test_restore_backup_from_future(self): """more complex test_restore_chain()""" fname = self.id().split('.')[3] From 24a1036037d7c7e05e4224935128309f2c9532c7 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 5 Sep 2022 02:57:40 +0300 Subject: [PATCH 07/55] [PBCKP-258] fix tests.ptrack.PtrackTest.test_ptrack_unprivileged --- tests/ptrack.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index d46ece11..783d3b3e 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -545,8 +545,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -583,9 +582,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -620,9 +617,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") node.safe_psql( "backupdb", @@ -641,7 +636,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') self.backup_node( backup_dir, 'node', node, From b8c2076437514d31153488d774fcf2aa2ef8b6ab Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 5 Sep 2022 03:04:18 +0300 Subject: [PATCH 08/55] [PBCKP-261] fix configure flags for tests.pgpro2068.BugTest.test_minrecpoint_on_replica --- travis/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 37614f97..1823b05d 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -47,7 +47,7 @@ cd postgres # Go to postgres dir if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then git apply -3 ../ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff fi -CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls +CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls --with-python make -s -j$(nproc) install #make -s -j$(nproc) -C 'src/common' install #make -s -j$(nproc) -C 'src/port' install From a4a2abde5295ae359413dc3353b8e25e743006f9 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 5 Sep 2022 17:12:58 +0300 Subject: [PATCH 09/55] [PBCKP-178] fix rare 'buffer error' in tests.validate.ValidateTest.test_validate_corrupt_page_header_map --- tests/validate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/validate.py b/tests/validate.py index 22a03c3b..966ad81a 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -4017,9 +4017,9 @@ class ValidateTest(ProbackupTest, unittest.TestCase): "Output: {0} \n CMD: {1}".format( self.output, self.cmd)) except ProbackupException as e: - self.assertTrue( - 'WARNING: An error occured during metadata decompression' in e.message and - 'data error' in e.message, + self.assertRegex( + e.message, + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 25fc034509d22af508559b38fbab847ccf8db577 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 5 Sep 2022 19:38:14 +0300 Subject: [PATCH 10/55] [PBCKP-236] stable test failure, dirty version --- tests/compatibility.py | 72 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/compatibility.py b/tests/compatibility.py index e274c22b..262b940e 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -10,6 +10,78 @@ module_name = 'compatibility' class CompatibilityTest(ProbackupTest, unittest.TestCase): + def setUp(self): + self.fname = self.id().split('.')[3] + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_catchup_with_different_remote_major_pg(self): + "Decription in jira issue PBCKP-236" #TODO REVIEW XXX explain the test + self.verbose = True + self.remote = True + pg_config = os.environ['PG_CONFIG'] + pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.9.6/bin/' + pg_config_ee_9_6 = pg_path_ee_9_6 + 'pg_config' + probackup_path_ee_9_6 = pg_path_ee_9_6 + 'pg_probackup' + pg_path_ee_11 = '/home/avaness/postgres/postgres.build.11/bin/' + pg_config_ee_11 = pg_path_ee_11 + 'pg_config' + probackup_path_ee_11 = pg_path_ee_11 + 'pg_probackup' + + os.environ['PG_CONFIG'] = pg_config_ee_11 + self.probackup_path = probackup_path_ee_11 + # os.environ['PG_CONFIG'] = pg_config_ee_9_6 + # self.probackup_path = probackup_path_ee_9_6 + + # backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + src_pg = self.make_simple_node( + base_dir=os.path.join(module_name, self.fname, 'src'), + set_replication=True, + # initdb_params=['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + + # do full catchup + os.environ['PG_CONFIG'] = pg_config_ee_11 + self.probackup_path = probackup_path_ee_11 + + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + # dst_pg = self.make_simple_node( + # base_dir=os.path.join(module_name, self.fname, 'dst'), + # set_replication=True, + # # initdb_params=['--data-checksums'] + # ) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream']#, '--remote-path=' + pg_path_ee_9_6] + ) + + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question2 AS SELECT 42 AS answer") + + # do delta catchup + #TODO REVIEW XXX try to apply only one catchup (FULL) for test failure + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_ee_9_6] + ) + + # Clean after yourself + self.del_test_dir(module_name, self.fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_page(self): From 9e9509d8aab21565d95b44daeafcca6b7516597c Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 8 Sep 2022 09:15:24 +0300 Subject: [PATCH 11/55] [PBCKP-263] fix for tests.archive.ArchiveTest.test_archive_get_batching_sanity (#532) --- src/archive.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/archive.c b/src/archive.c index 48114d95..1a19c3d8 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1263,6 +1263,7 @@ uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, arg->thread_num = i+1; arg->files = batch_files; + arg->n_fetched = 0; } /* Run threads */ From d4d78e18f34ee6d7bf04253a5da597165c8134b6 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 9 Sep 2022 22:09:41 +0300 Subject: [PATCH 12/55] [PBCKP-277] stabilize catchup.CatchupTest.test_config_exclusion --- .travis.yml | 24 +++++++++++++++--------- tests/catchup.py | 4 ++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7dae2ed..52d6dba1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,17 +26,23 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE - - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE +# - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master +# - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE +# - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE +# - PG_VERSION=10 PG_BRANCH=REL_10_STABLE +# - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE +# - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=10 PG_BRANCH=REL_10_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE MODE=catchup.CatchupTest.test_config_exclusion # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=compression # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta diff --git a/tests/catchup.py b/tests/catchup.py index a83755c5..7ecd8469 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -1362,6 +1362,7 @@ class CatchupTest(ProbackupTest, unittest.TestCase): dst_options = {} dst_options['port'] = str(dst_pg.port) self.set_auto_conf(dst_pg, dst_options) + dst_pg._assign_master(src_pg) dst_pg.slow_start(replica = True) dst_pg.stop() @@ -1390,6 +1391,7 @@ class CatchupTest(ProbackupTest, unittest.TestCase): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_pg.catchup() # wait for replication dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') @@ -1419,6 +1421,7 @@ class CatchupTest(ProbackupTest, unittest.TestCase): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(2*42)") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_pg.catchup() # wait for replication dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') @@ -1447,6 +1450,7 @@ class CatchupTest(ProbackupTest, unittest.TestCase): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(3*42)") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_pg.catchup() # wait for replication dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') From 42241bd3ba8f2ee2d8d473637a3a3f143ba8eb3e Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 9 Sep 2022 22:21:28 +0300 Subject: [PATCH 13/55] [PBCKP-277] fix .travis.yml typo --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 52d6dba1..7cf50d0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,10 +36,10 @@ env: # - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE MODE=catchup.CatchupTest.test_config_exclusion - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE MODE=catchup.CatchupTest.test_config_exclusion - PG_VERSION=10 PG_BRANCH=REL_10_STABLE MODE=catchup.CatchupTest.test_config_exclusion - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE MODE=catchup.CatchupTest.test_config_exclusion - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE MODE=catchup.CatchupTest.test_config_exclusion From f78c63c8f56b8a7064f8799460156001dd1c6a76 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 11 Sep 2022 04:14:18 +0300 Subject: [PATCH 14/55] [PBCKP-236] first-stage compatibility protocol impl with stubs --- src/pg_probackup.h | 8 ++++++-- src/utils/file.c | 20 +++++++++++++++++--- src/utils/file.h | 2 +- src/utils/remote.c | 34 +++++++++++++++++++++++++++++----- 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1885a191..e68afc57 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -341,8 +341,8 @@ typedef enum ShowFormat #define PROGRAM_VERSION "2.5.8" /* update when remote agent API or behaviour changes */ -#define AGENT_PROTOCOL_VERSION 20501 -#define AGENT_PROTOCOL_VERSION_STR "2.5.1" +#define AGENT_PROTOCOL_VERSION 20509 +#define AGENT_PROTOCOL_VERSION_STR "2.5.9" /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" @@ -881,6 +881,10 @@ extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, IncrRestoreMode incremental_mode); +/* in remote.c */ +extern void check_remote_agent_compatibility(int agent_version, char *compatibility_str); +extern size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size); + /* in merge.c */ extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync); extern void merge_backups(pgBackup *backup, pgBackup *next_backup); diff --git a/src/utils/file.c b/src/utils/file.c index 7103c8f1..e3d1e580 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -268,9 +268,10 @@ fio_write_all(int fd, void const* buf, size_t size) return offs; } +//TODO REVIEW XXX move to remote.c???? /* Get version of remote agent */ -int -fio_get_agent_version(void) +void +fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) { fio_header hdr; hdr.cop = FIO_AGENT_VERSION; @@ -278,8 +279,13 @@ fio_get_agent_version(void) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size > payload_buf_size) + { + elog(ERROR, "Bad protocol, insufficient payload_buf_size=%u", payload_buf_size); + } - return hdr.arg; + *protocol = hdr.arg; + IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size); } /* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ @@ -3210,6 +3216,7 @@ fio_delete_impl(mode_t mode, char *buf) } /* Execute commands at remote host */ +//TODO REVIEW XXX move to remote.c? void fio_communicate(int in, int out) { @@ -3316,6 +3323,13 @@ fio_communicate(int in, int out) case FIO_AGENT_VERSION: hdr.arg = AGENT_PROTOCOL_VERSION; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + //TODO REVIEW XXX is it allowed by ANSI C to declare new scope inside??? + { + size_t payload_size = prepare_remote_agent_compatibility_str(buf, buf_size); + IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); + //TODO REVIEW XXX make INFO to LOG or VERBOSE + elog(INFO, "TODO REVIEW XXX sent agent compatibility\n %s", buf); + } break; case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); diff --git a/src/utils/file.h b/src/utils/file.h index a554b4ab..92c5f2ea 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -91,7 +91,7 @@ extern fio_location MyLocation; extern void fio_redirect(int in, int out, int err); extern void fio_communicate(int in, int out); -extern int fio_get_agent_version(void); +extern void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); extern ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg); diff --git a/src/utils/remote.c b/src/utils/remote.c index 046ebd81..c7a1f933 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -117,6 +117,9 @@ bool launch_agent(void) int infd[2]; int errfd[2]; int agent_version; + //TODO REVIEW XXX review buf_size + size_t payload_buf_size = 1024 * 8; + char payload_buf[payload_buf_size]; ssh_argc = 0; #ifdef WIN32 @@ -238,10 +241,31 @@ bool launch_agent(void) fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */ } - /* Make sure that remote agent has the same version - * TODO: we must also check PG version and fork edition + /* Make sure that remote agent has the same version, fork and other features to be binary compatible */ - agent_version = fio_get_agent_version(); + fio_get_agent_version(&agent_version, payload_buf, payload_buf_size); + check_remote_agent_compatibility(0, payload_buf); + + return true; +} + +//TODO REVIEW XXX review macro +#define STR(macro) #macro +size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size) +{ + size_t payload_size = snprintf(compatibility_buf, buf_size, + "%s\n%s\n%s\n%s\n", + STR(PG_MAJORVERSION), PG_MAJORVERSION, + STR(PGPRO_EDN), PGPRO_EDN); + if (payload_size >= buf_size) + { + elog(ERROR, "TODO REVIEW XXX too bad message buffer exhaust"); + } + return payload_size + 1; +} + +void check_remote_agent_compatibility(int agent_version, char *compatibility_str) +{ if (agent_version != AGENT_PROTOCOL_VERSION) { char agent_version_str[1024]; @@ -254,6 +278,6 @@ bool launch_agent(void) "consider to upgrade pg_probackup binary", agent_version_str, AGENT_PROTOCOL_VERSION_STR); } - - return true; + assert(false); + elog(ERROR, " check_remote_agent_compatibility() not implemented"); } From 1dfa5b99c2c20a1a97d01b7f141d450a938a9a4f Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 11 Sep 2022 04:37:46 +0300 Subject: [PATCH 15/55] [PBCKP-236] draft, first-stage compatibility protocol impl with stubs --- src/utils/file.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/file.c b/src/utils/file.c index e3d1e580..6c7bdbbf 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -3323,6 +3323,7 @@ fio_communicate(int in, int out) case FIO_AGENT_VERSION: hdr.arg = AGENT_PROTOCOL_VERSION; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + assert(false); //TODO REVIEW XXX is it allowed by ANSI C to declare new scope inside??? { size_t payload_size = prepare_remote_agent_compatibility_str(buf, buf_size); From c3d3c026c2f8446b93e5e73150a3c55a153f4317 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 11 Sep 2022 04:44:18 +0300 Subject: [PATCH 16/55] [PBCKP-236] draft, first-stage compatibility protocol impl with stubs --- src/utils/remote.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index c7a1f933..6a6c3c12 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -255,8 +255,8 @@ size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t bu { size_t payload_size = snprintf(compatibility_buf, buf_size, "%s\n%s\n%s\n%s\n", - STR(PG_MAJORVERSION), PG_MAJORVERSION, - STR(PGPRO_EDN), PGPRO_EDN); + STR(PG_MAJORVERSION), PG_MAJORVERSION); +// STR(PGPRO_EDN), PGPRO_EDN); if (payload_size >= buf_size) { elog(ERROR, "TODO REVIEW XXX too bad message buffer exhaust"); From 46b7079edd63b43c41c104830d7feef9d4536476 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 11 Sep 2022 04:46:15 +0300 Subject: [PATCH 17/55] [PBCKP-236] draft, first-stage compatibility protocol impl with stubs --- src/utils/remote.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 6a6c3c12..e4963b62 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -254,7 +254,8 @@ bool launch_agent(void) size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size) { size_t payload_size = snprintf(compatibility_buf, buf_size, - "%s\n%s\n%s\n%s\n", +// "%s\n%s\n%s\n%s\n", + "%s\n%s\n", STR(PG_MAJORVERSION), PG_MAJORVERSION); // STR(PGPRO_EDN), PGPRO_EDN); if (payload_size >= buf_size) From f5fde7ef8e1ee479932df78086f7bed817c53902 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 12 Sep 2022 02:46:29 +0300 Subject: [PATCH 18/55] [PBCKP-236] draft, first-stage compatibility protocol impl with stubs --- src/utils/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 6c7bdbbf..65d0699c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -281,7 +281,7 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > payload_buf_size) { - elog(ERROR, "Bad protocol, insufficient payload_buf_size=%u", payload_buf_size); + elog(ERROR, "Bad protocol, insufficient payload_buf_size=%zu", payload_buf_size); } *protocol = hdr.arg; @@ -3323,7 +3323,6 @@ fio_communicate(int in, int out) case FIO_AGENT_VERSION: hdr.arg = AGENT_PROTOCOL_VERSION; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - assert(false); //TODO REVIEW XXX is it allowed by ANSI C to declare new scope inside??? { size_t payload_size = prepare_remote_agent_compatibility_str(buf, buf_size); @@ -3331,6 +3330,7 @@ fio_communicate(int in, int out) //TODO REVIEW XXX make INFO to LOG or VERBOSE elog(INFO, "TODO REVIEW XXX sent agent compatibility\n %s", buf); } + assert(false); break; case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); From 6d3ad888cd00dbbdac734c316970ecec6f0e123c Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Tue, 21 Jun 2022 12:54:30 +0300 Subject: [PATCH 19/55] [PBCKP-125] changes in function call CreateWalDirectoryMethod for 15 version Tags: pg_probackup --- src/stream.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/stream.c b/src/stream.c index 1ee8dee3..3b947af0 100644 --- a/src/stream.c +++ b/src/stream.c @@ -274,7 +274,13 @@ StreamLog(void *arg) ctl.synchronous = false; ctl.mark_done = false; -#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 150000 + ctl.walmethod = CreateWalDirectoryMethod( + stream_arg->basedir, + COMPRESSION_NONE, + 0, + false); +#elif PG_VERSION_NUM >= 100000 ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, // (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, From 53abc0b6e735944354b2c40483b9ac82bcc4c499 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Fri, 1 Jul 2022 15:46:53 +0300 Subject: [PATCH 20/55] [PGPRO-6938] pg_probackup has been ported to version 15 Has been tested on 15beta2 and 16 tags: pg_probackup --- src/backup.c | 34 +++--- src/parsexlog.c | 14 ++- src/stream.c | 2 +- src/utils/configuration.c | 1 + tests/archive.py | 13 ++- tests/auth_test.py | 105 +++++++++++++----- tests/backup.py | 186 +++++++++++++++++++++++++++----- tests/false_positive.py | 3 + tests/helpers/ptrack_helpers.py | 20 +++- tests/ptrack.py | 46 +++++++- tests/replica.py | 1 + tests/restore.py | 47 +++++++- tests/retention.py | 2 + 13 files changed, 379 insertions(+), 95 deletions(-) diff --git a/src/backup.c b/src/backup.c index 03ff7b72..0fa8ee9f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1056,20 +1056,14 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, uint32 lsn_lo; params[0] = label; - elog(INFO, "wait for pg_start_backup()"); + elog(INFO, "wait for pg_backup_start()"); /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; - if (!exclusive_backup) - res = pgut_execute(conn, - "SELECT pg_catalog.pg_start_backup($1, $2, false)", - 2, - params); - else - res = pgut_execute(conn, - "SELECT pg_catalog.pg_start_backup($1, $2)", - 2, - params); + res = pgut_execute(conn, + "SELECT pg_catalog.pg_backup_start($1, $2)", + 2, + params); /* * Set flag that pg_start_backup() was called. If an error will happen it @@ -1618,7 +1612,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," " current_timestamp(0)::timestamptz," - " pg_catalog.pg_stop_backup() as lsn", + " pg_catalog.pg_backup_stop() as lsn", stop_backup_on_master_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1626,7 +1620,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " lsn," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false, false)", + " FROM pg_catalog.pg_backup_stop(false)", stop_backup_on_master_before10_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1634,7 +1628,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " lsn," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)", + " FROM pg_catalog.pg_backup_stop()", /* * In case of backup from replica >= 9.6 we do not trust minRecPoint * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. @@ -1646,7 +1640,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " pg_catalog.pg_last_wal_replay_lsn()," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false, false)", + " FROM pg_catalog.pg_backup_stop(false)", stop_backup_on_replica_before10_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1654,7 +1648,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " pg_catalog.pg_last_xlog_replay_location()," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)"; + " FROM pg_catalog.pg_backup_stop()"; const char * const stop_backup_query = is_exclusive ? @@ -1682,7 +1676,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica */ sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); if (!sent) - elog(ERROR, "Failed to send pg_stop_backup query"); + elog(ERROR, "Failed to send pg_backup_stop query"); /* After we have sent pg_stop_backup, we don't need this callback anymore */ pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_params); @@ -1728,7 +1722,7 @@ pg_stop_backup_consume(PGconn *conn, int server_version, if (interrupted) { pgut_cancel(conn); - elog(ERROR, "interrupted during waiting for pg_stop_backup"); + elog(ERROR, "interrupted during waiting for pg_backup_stop"); } if (pg_stop_backup_timeout == 1) @@ -1741,7 +1735,7 @@ pg_stop_backup_consume(PGconn *conn, int server_version, if (pg_stop_backup_timeout > timeout) { pgut_cancel(conn); - elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); + elog(ERROR, "pg_backup_stop doesn't answer in %d seconds, cancel it", timeout); } } else @@ -1753,7 +1747,7 @@ pg_stop_backup_consume(PGconn *conn, int server_version, /* Check successfull execution of pg_stop_backup() */ if (!query_result) - elog(ERROR, "pg_stop_backup() failed"); + elog(ERROR, "pg_backup_stop() failed"); else { switch (PQresultStatus(query_result)) diff --git a/src/parsexlog.c b/src/parsexlog.c index 7f1ca9c7..5cf76031 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -29,7 +29,10 @@ * RmgrNames is an array of resource manager names, to make error messages * a bit nicer. */ -#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 150000 +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ + name, +#elif PG_VERSION_NUM >= 100000 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ name, #else @@ -1769,7 +1772,8 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, /* Is this a special record type that I recognize? */ - if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE) + if (rmid == RM_DBASE_ID + && (rminfo == XLOG_DBASE_CREATE_WAL_LOG || rminfo == XLOG_DBASE_CREATE_FILE_COPY)) { /* * New databases can be safely ignored. They would be completely @@ -1823,13 +1827,13 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, RmgrNames[rmid], info); } - for (block_id = 0; block_id <= record->max_block_id; block_id++) + for (block_id = 0; block_id <= record->record->max_block_id; block_id++) { RelFileNode rnode; ForkNumber forknum; BlockNumber blkno; - if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) + if (!XLogRecGetBlockTagExtended(record, block_id, &rnode, &forknum, &blkno, NULL)) continue; /* We only care about the main fork; others are copied as is */ @@ -1946,4 +1950,4 @@ static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *r #else return XLogReaderAllocate(&SimpleXLogPageRead, reader_data); #endif -} \ No newline at end of file +} diff --git a/src/stream.c b/src/stream.c index 3b947af0..7735f35f 100644 --- a/src/stream.c +++ b/src/stream.c @@ -277,7 +277,7 @@ StreamLog(void *arg) #if PG_VERSION_NUM >= 150000 ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, - COMPRESSION_NONE, + PG_COMPRESSION_NONE, 0, false); #elif PG_VERSION_NUM >= 100000 diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 7ab242aa..98c3b299 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -22,6 +22,7 @@ #include #endif #include +#include #define MAXPG_LSNCOMPONENT 8 diff --git a/tests/archive.py b/tests/archive.py index 52fb225e..81d013f6 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -250,6 +250,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): "--log-level-file=LOG"], gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() @@ -314,6 +315,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): "--log-level-file=info"], gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() @@ -341,9 +343,14 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): with open(log_file, 'r') as f: log_content = f.read() - self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", - log_content) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + log_content) + else: + self.assertIn( + "ERROR: pg_backup_stop doesn't answer in 60 seconds, cancel it", + log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: diff --git a/tests/auth_test.py b/tests/auth_test.py index 78af21be..39786d7a 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -51,16 +51,29 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase): 1, 0, "Expecting Error due to missing grant on EXECUTE.") except ProbackupException as e: - self.assertIn( - "ERROR: query failed: ERROR: permission denied " - "for function pg_start_backup", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_start_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_backup_start", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION" - " pg_start_backup(text, boolean, boolean) TO backup;") + if self.get_version(node) < 150000: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_start_backup(text, boolean, boolean) TO backup;") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_backup_start(text, boolean) TO backup;") if self.get_version(node) < 100000: node.safe_psql( @@ -97,17 +110,24 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase): 1, 0, "Expecting Error due to missing grant on EXECUTE.") except ProbackupException as e: - self.assertIn( - "ERROR: query failed: ERROR: permission denied " - "for function pg_stop_backup", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_stop_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_backup_stop", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) if self.get_version(node) < self.version_to_num('10.0'): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") - else: + elif self.get_vestion(node) < self.version_to_num('15.0'): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION " @@ -116,6 +136,16 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION " + "pg_backup_stop(boolean) TO backup") + # Do this for ptrack backups + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup") + self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -177,20 +207,37 @@ class AuthTest(unittest.TestCase): except StartNodeException: raise unittest.skip("Node hasn't started") - cls.node.safe_psql( - "postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + if cls.pb.get_version(cls.node) < 150000: + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + else: + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') @classmethod diff --git a/tests/backup.py b/tests/backup.py index 0cba8fe7..4f447c9b 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1927,9 +1927,10 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -1964,7 +1965,46 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if self.ptrack: node.safe_psql( @@ -2266,9 +2306,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -2282,7 +2324,28 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "BEGIN; " + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) # enable STREAM backup node.safe_psql( @@ -3054,8 +3117,8 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -3075,7 +3138,31 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if ProbackupTest.enterprise: node.safe_psql( @@ -3183,9 +3270,10 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -3205,7 +3293,31 @@ class BackupTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # > 15 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if ProbackupTest.enterprise: node.safe_psql( @@ -3331,7 +3443,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): log_content) self.assertIn( - 'FROM pg_catalog.pg_stop_backup', + 'FROM pg_catalog.pg_backup_stop', log_content) self.assertIn( @@ -3369,10 +3481,15 @@ class BackupTest(ProbackupTest, unittest.TestCase): node.safe_psql( 'postgres', 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) FROM backup') - else: + elif self.get_version(node) < 150000: node.safe_psql( 'postgres', 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup') + else: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) FROM backup') + # Full backup in streaming mode try: @@ -3380,17 +3497,32 @@ class BackupTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--stream', '-U', 'backup']) # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions on pg_stop_backup " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + if self.get_version(node) < 150000: + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_stop_backup " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + else: + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_backup_stop " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "ERROR: permission denied for function pg_stop_backup", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: permission denied for function pg_stop_backup", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: permission denied for function pg_backup_stop", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + self.assertIn( "query was: SELECT pg_catalog.txid_snapshot_xmax", e.message, diff --git a/tests/false_positive.py b/tests/false_positive.py index a101f810..2ededdf1 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -198,6 +198,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -257,6 +258,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--log-level-console=LOG'], gdb=True) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -308,6 +310,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, gdb=True, options=['--stream', '--log-level-file=LOG']) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 59eb12ae..d800f0d3 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -476,8 +476,8 @@ class ProbackupTest(object): 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) - # >= 10 - else: + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'postgres', 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' @@ -492,6 +492,22 @@ class ProbackupTest(object): 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) + # >= 15 + else: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): res = node.execute( diff --git a/tests/ptrack.py b/tests/ptrack.py index 783d3b3e..a01405d6 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -582,9 +582,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -617,7 +618,44 @@ class PtrackTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) node.safe_psql( "backupdb", diff --git a/tests/replica.py b/tests/replica.py index 24dbaa39..ea69e2d0 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -775,6 +775,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): '--stream'], gdb=True) + # Attention! this breakpoint is set to a probackup internal function, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() diff --git a/tests/restore.py b/tests/restore.py index ae1c7cbe..49538bd1 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3272,9 +3272,10 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -3308,7 +3309,45 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if self.ptrack: # TODO why backup works without these grants ? diff --git a/tests/retention.py b/tests/retention.py index b0399a23..122ab28a 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1519,6 +1519,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): gdb = self.backup_node( backup_dir, 'node', node, backup_type='page', gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -1568,6 +1569,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): gdb = self.backup_node( backup_dir, 'node', node, backup_type='page', gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb._execute('signal SIGKILL') From 0a1a075b12a466b17be6f1f9fcc8895e5a7e247f Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Tue, 23 Aug 2022 20:40:42 +0300 Subject: [PATCH 21/55] [PGPRO-6938] macro conditions fixed tags: pg_probackup --- .travis.yml | 1 + README.md | 76 ++++++++++++++++++++++----------------------- doc/pgprobackup.xml | 23 +++++++++++++- src/backup.c | 74 ++++++++++++++++++++++++++++++++++--------- src/parsexlog.c | 12 +++++++ src/pg_probackup.h | 6 ++++ 6 files changed, 139 insertions(+), 53 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7dae2ed..f113d05c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=REL_15_STABLE - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE diff --git a/README.md b/README.md index 5da8d199..43397847 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.6, 10, 11, 12, 13, 14; +* PostgreSQL 9.6, 10, 11, 12, 13, 14, 15; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -41,9 +41,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## ptrack support `PTRACK` backup support provided via following options: -* vanilla PostgreSQL 11, 12, 13, 14 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13 -* Postgres Pro Enterprise 11, 12, 13 +* vanilla PostgreSQL 11, 12, 13, 14, 15 with [ptrack extension](https://github.com/postgrespro/ptrack) +* Postgres Pro Standard 11, 12, 13, 14 +* Postgres Pro Enterprise 11, 12, 13, 14 ## Limitations @@ -74,62 +74,62 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{14,13,12,11,10,9.6} +sudo apt-get source pg-probackup-{15,14,13,12,11,10,9.6} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}{-dbg,} +sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{14,13,12,11,10,9.6} +yumdownloader --source pg_probackup-{15,14,13,12,11,10,9.6} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{14,13,12,11,10,9.6} -zypper install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11,10,9.6} +zypper install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{14,13,12,11,10,9.6} +zypper si pg_probackup-{15,14,13,12,11,10,9.6} #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo ``` #### pg_probackup for PostgresPro Standard and Enterprise @@ -137,8 +137,8 @@ sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' @@ -148,35 +148,35 @@ sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index fc2a341e..6babf00f 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -653,7 +653,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; - For PostgreSQL 10 or higher: + For PostgreSQL 10: BEGIN; @@ -672,6 +672,27 @@ GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; + + + For PostgreSQL 15 or higher: + + +BEGIN; +CREATE ROLE backup WITH LOGIN; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; +COMMIT; In the diff --git a/src/backup.c b/src/backup.c index 0fa8ee9f..31289978 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1056,14 +1056,22 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, uint32 lsn_lo; params[0] = label; +#if PG_VERSION_NUM >= 150000 elog(INFO, "wait for pg_backup_start()"); +#else + elog(INFO, "wait for pg_start_backup()"); +#endif /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; res = pgut_execute(conn, - "SELECT pg_catalog.pg_backup_start($1, $2)", - 2, - params); +#if PG_VERSION_NUM >= 150000 + "SELECT pg_catalog.pg_backup_start($1, $2)", +#else + "SELECT pg_catalog.pg_start_backup($1, $2, false)", +#endif + 2, + params); /* * Set flag that pg_start_backup() was called. If an error will happen it @@ -1612,7 +1620,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," " current_timestamp(0)::timestamptz," - " pg_catalog.pg_backup_stop() as lsn", + " pg_catalog.pg_stop_backup() as lsn", stop_backup_on_master_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1620,7 +1628,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " lsn," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_backup_stop(false)", + " FROM pg_catalog.pg_stop_backup(false, false)", stop_backup_on_master_before10_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1628,7 +1636,15 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " lsn," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_backup_stop()", + " FROM pg_catalog.pg_stop_backup(false)", + stop_backup_on_master_after15_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_backup_stop(false)", /* * In case of backup from replica >= 9.6 we do not trust minRecPoint * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. @@ -1640,7 +1656,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " pg_catalog.pg_last_wal_replay_lsn()," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_backup_stop(false)", + " FROM pg_catalog.pg_stop_backup(false, false)", stop_backup_on_replica_before10_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1648,19 +1664,33 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " pg_catalog.pg_last_xlog_replay_location()," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_backup_stop()"; + " FROM pg_catalog.pg_stop_backup(false)", + stop_backup_on_replica_after15_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_last_wal_replay_lsn()," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_backup_stop(false)"; const char * const stop_backup_query = is_exclusive ? stop_exlusive_backup_query : - server_version >= 100000 ? + server_version >= 150000 ? (is_started_on_replica ? - stop_backup_on_replica_query : - stop_backup_on_master_query + stop_backup_on_replica_after15_query : + stop_backup_on_master_after15_query ) : - (is_started_on_replica ? - stop_backup_on_replica_before10_query : - stop_backup_on_master_before10_query + (server_version >= 100000 ? + (is_started_on_replica ? + stop_backup_on_replica_query : + stop_backup_on_master_query + ) : + (is_started_on_replica ? + stop_backup_on_replica_before10_query : + stop_backup_on_master_before10_query + ) ); bool sent = false; @@ -1676,7 +1706,11 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica */ sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); if (!sent) +#if PG_VERSION_NUM >= 150000 elog(ERROR, "Failed to send pg_backup_stop query"); +#else + elog(ERROR, "Failed to send pg_stop_backup query"); +#endif /* After we have sent pg_stop_backup, we don't need this callback anymore */ pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_params); @@ -1722,7 +1756,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version, if (interrupted) { pgut_cancel(conn); +#if PG_VERSION_NUM >= 150000 elog(ERROR, "interrupted during waiting for pg_backup_stop"); +#else + elog(ERROR, "interrupted during waiting for pg_stop_backup"); +#endif } if (pg_stop_backup_timeout == 1) @@ -1735,7 +1773,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version, if (pg_stop_backup_timeout > timeout) { pgut_cancel(conn); +#if PG_VERSION_NUM >= 150000 elog(ERROR, "pg_backup_stop doesn't answer in %d seconds, cancel it", timeout); +#else + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); +#endif } } else @@ -1747,7 +1789,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version, /* Check successfull execution of pg_stop_backup() */ if (!query_result) +#if PG_VERSION_NUM >= 150000 elog(ERROR, "pg_backup_stop() failed"); +#else + elog(ERROR, "pg_stop_backup() failed"); +#endif else { switch (PQresultStatus(query_result)) diff --git a/src/parsexlog.c b/src/parsexlog.c index 5cf76031..f12aae90 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1773,7 +1773,11 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, /* Is this a special record type that I recognize? */ if (rmid == RM_DBASE_ID +#if PG_VERSION_NUM >= 150000 && (rminfo == XLOG_DBASE_CREATE_WAL_LOG || rminfo == XLOG_DBASE_CREATE_FILE_COPY)) +#else + && rminfo == XLOG_DBASE_CREATE) +#endif { /* * New databases can be safely ignored. They would be completely @@ -1827,13 +1831,21 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, RmgrNames[rmid], info); } +#if PG_VERSION_NUM >= 150000 for (block_id = 0; block_id <= record->record->max_block_id; block_id++) +#else + for (block_id = 0; block_id <= record->max_block_id; block_id++) +#endif { RelFileNode rnode; ForkNumber forknum; BlockNumber blkno; +#if PG_VERSION_NUM >= 150000 if (!XLogRecGetBlockTagExtended(record, block_id, &rnode, &forknum, &blkno, NULL)) +#else + if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) +#endif continue; /* We only care about the main fork; others are copied as is */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1885a191..533b05d5 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -50,6 +50,12 @@ #include #endif +#if PG_VERSION_NUM >= 150000 +// _() is explicitly undefined in libpq-int.h +// https://github.com/postgres/postgres/commit/28ec316787674dd74d00b296724a009b6edc2fb0 +#define _(s) gettext(s) +#endif + /* Wrap the code that we're going to delete after refactoring in this define*/ #define REFACTORE_ME From 497751c0b63b8cfdf825beeec26f1d66a902be6e Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 15 Sep 2022 05:47:36 +0300 Subject: [PATCH 22/55] [PBCKP-236] draft, tests.CompatibilityTest.test_catchup_with_different_remote_major_pg fixes --- tests/compatibility.py | 48 ++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 262b940e..0562b441 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,29 +14,30 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): - "Decription in jira issue PBCKP-236" #TODO REVIEW XXX explain the test + """ + Decription in jira issue PBCKP-236 + This test requires builds both PGPROEE11 and PGPROEE9_6 + + prerequisites: + - git tag for PBCKP 2.5.1 + - master probackup build should be inside PGPROEE11 + - agent probackup build is inside PGPROEE9_6 + + calling probackup PGPROEE9_6 agent from PGPROEE11 probackup master for DELTA backup causes the PBCKP-236 problem + + please correct path for agent's pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + """ + self.verbose = True self.remote = True - pg_config = os.environ['PG_CONFIG'] - pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.9.6/bin/' - pg_config_ee_9_6 = pg_path_ee_9_6 + 'pg_config' - probackup_path_ee_9_6 = pg_path_ee_9_6 + 'pg_probackup' - pg_path_ee_11 = '/home/avaness/postgres/postgres.build.11/bin/' - pg_config_ee_11 = pg_path_ee_11 + 'pg_config' - probackup_path_ee_11 = pg_path_ee_11 + 'pg_probackup' + # please use your own local path + pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' - os.environ['PG_CONFIG'] = pg_config_ee_11 - self.probackup_path = probackup_path_ee_11 - # os.environ['PG_CONFIG'] = pg_config_ee_9_6 - # self.probackup_path = probackup_path_ee_9_6 - - # backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') src_pg = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'src'), set_replication=True, - # initdb_params=['--data-checksums'] ) src_pg.slow_start() src_pg.safe_psql( @@ -44,20 +45,12 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): "CREATE TABLE ultimate_question AS SELECT 42 AS answer") # do full catchup - os.environ['PG_CONFIG'] = pg_config_ee_11 - self.probackup_path = probackup_path_ee_11 - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) - # dst_pg = self.make_simple_node( - # base_dir=os.path.join(module_name, self.fname, 'dst'), - # set_replication=True, - # # initdb_params=['--data-checksums'] - # ) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, - options=['-d', 'postgres', '-p', str(src_pg.port), '--stream']#, '--remote-path=' + pg_path_ee_9_6] + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream'] ) dst_options = {} @@ -70,12 +63,13 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): "postgres", "CREATE TABLE ultimate_question2 AS SELECT 42 AS answer") - # do delta catchup - #TODO REVIEW XXX try to apply only one catchup (FULL) for test failure + # do delta catchup with remote pg_probackup agent with another postgres major version + # this DELTA backup should fail without PBCKP-236 patch. self.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, + # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_ee_9_6] ) From 044c0376c849608cb79a0e1d53863a619c4799c5 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Thu, 15 Sep 2022 15:45:47 +0300 Subject: [PATCH 23/55] [PBCKP-125] changes in function call CreateWalDirectoryMethod for 15 version --- src/stream.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stream.c b/src/stream.c index 7735f35f..f7bbeae5 100644 --- a/src/stream.c +++ b/src/stream.c @@ -274,18 +274,20 @@ StreamLog(void *arg) ctl.synchronous = false; ctl.mark_done = false; +#if PG_VERSION_NUM >= 100000 #if PG_VERSION_NUM >= 150000 ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, PG_COMPRESSION_NONE, 0, false); -#elif PG_VERSION_NUM >= 100000 +#else /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 150000 */ ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, // (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, 0, false); +#endif /* PG_VERSION_NUM >= 150000 */ ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; ctl.do_sync = false; /* We sync all files at the end of backup */ From eefd88768a8cb2b6350d69ca47cc28a5e8beb160 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 01:06:52 +0300 Subject: [PATCH 24/55] [PBCKP-236] draft, solution without couple of unapplied shortenings --- src/pg_probackup.h | 5 ++- src/utils/file.c | 16 ++++--- src/utils/remote.c | 103 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 99 insertions(+), 25 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e68afc57..13dfe198 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -882,8 +882,9 @@ extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgda IncrRestoreMode incremental_mode); /* in remote.c */ -extern void check_remote_agent_compatibility(int agent_version, char *compatibility_str); -extern size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size); +extern void check_remote_agent_compatibility(int agent_version, + char *compatibility_str, size_t compatibility_str_max_size); +extern size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size); /* in merge.c */ extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync); diff --git a/src/utils/file.c b/src/utils/file.c index 65d0699c..b0dc39ae 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -281,7 +281,8 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > payload_buf_size) { - elog(ERROR, "Bad protocol, insufficient payload_buf_size=%zu", payload_buf_size); + //TODO REVIEW XXX %zu is C99 but not ANSI S standard, should we cast to unsigned long? + elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size); } *protocol = hdr.arg; @@ -3321,17 +3322,18 @@ fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; case FIO_AGENT_VERSION: - hdr.arg = AGENT_PROTOCOL_VERSION; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - //TODO REVIEW XXX is it allowed by ANSI C to declare new scope inside??? { - size_t payload_size = prepare_remote_agent_compatibility_str(buf, buf_size); + size_t payload_size = prepare_compatibility_str(buf, buf_size); + + hdr.arg = AGENT_PROTOCOL_VERSION; + hdr.size = payload_size; + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); //TODO REVIEW XXX make INFO to LOG or VERBOSE elog(INFO, "TODO REVIEW XXX sent agent compatibility\n %s", buf); + break; } - assert(false); - break; case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st); diff --git a/src/utils/remote.c b/src/utils/remote.c index e4963b62..af3e460c 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -118,7 +118,7 @@ bool launch_agent(void) int errfd[2]; int agent_version; //TODO REVIEW XXX review buf_size - size_t payload_buf_size = 1024 * 8; + int payload_buf_size = 1024 * 8; char payload_buf[payload_buf_size]; ssh_argc = 0; @@ -244,29 +244,83 @@ bool launch_agent(void) /* Make sure that remote agent has the same version, fork and other features to be binary compatible */ fio_get_agent_version(&agent_version, payload_buf, payload_buf_size); - check_remote_agent_compatibility(0, payload_buf); + check_remote_agent_compatibility(agent_version, payload_buf, payload_buf_size); return true; } -//TODO REVIEW XXX review macro -#define STR(macro) #macro -size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size) +#define COMPATIBILITY_VAL(macro) #macro, macro +#define COMPATIBILITY_STR(macro) #macro +#define COMPATIBILITY_VAL_STR(macro) #macro, COMPATIBILITY_STR(macro) + +#define COMPATIBILITY_VAL_SEPARATOR "=" +#define COMPATIBILITY_LINE_SEPARATOR "\n" + +static char* compatibility_params[] = { + COMPATIBILITY_VAL(PG_MAJORVERSION), + //TODO remove? + //TODO doesn't work macro name check for ints!!!! + COMPATIBILITY_VAL_STR(SIZEOF_VOID_P), + //TODO REVIEW XXX can use edition.h/extract_pgpro_edition() +#ifdef PGPRO_EDN + //TODO add vanilla + //TODO make "1c" -> "vanilla" + COMPATIBILITY_VAL(PGPRO_EDN), +#endif +}; + +/* + * Compose compatibility string to be sent by pg_probackup agent + * through ssh and to be verified by pg_probackup peer. + * Compatibility string contains postgres essential vars as strings + * in format "var_name" + COMPATIBILITY_VAL_SEPARATOR + "var_value" + COMPATIBILITY_LINE_SEPARATOR + */ +size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { - size_t payload_size = snprintf(compatibility_buf, buf_size, -// "%s\n%s\n%s\n%s\n", - "%s\n%s\n", - STR(PG_MAJORVERSION), PG_MAJORVERSION); -// STR(PGPRO_EDN), PGPRO_EDN); - if (payload_size >= buf_size) + char tmp_buf[1024]; + int size_inc = 0; + size_t result_size = 1; + size_t compatibility_params_array_size = sizeof compatibility_params / sizeof compatibility_params[0];; + + *compatibility_buf = '\0'; + Assert(compatibility_params_array_size % 2 == 0); + + //TODO !!!! + for (int i = 0; i < compatibility_params_array_size; i+=2) { - elog(ERROR, "TODO REVIEW XXX too bad message buffer exhaust"); + size_inc = snprintf(compatibility_buf + size_inc, compatibility_buf_size, + "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, + compatibility_params[i], compatibility_params[i+1]); + +// size_inc = snprintf(tmp_buf, sizeof tmp_buf, +// "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, +// compatibility_params[i], compatibility_params[i+1]); + if (size_inc >= sizeof tmp_buf) + { + //TODO make Assert + elog(ERROR, "Compatibility params from agent doesn't fit to %zu chars, %s=%s", + sizeof tmp_buf - 1, compatibility_params[i], compatibility_params[i+1] ); + } + + result_size += size_inc; + if (result_size > compatibility_buf_size) + { + //TODO make Assert + elog(ERROR, "Can't fit compatibility string size %zu to buffer size %zu:\n%s\n%s", + result_size, compatibility_buf_size, compatibility_buf, tmp_buf); + } + strcat(compatibility_buf, tmp_buf); } - return payload_size + 1; + return result_size; } -void check_remote_agent_compatibility(int agent_version, char *compatibility_str) +/* + * Check incoming remote agent's compatibility params for equality to local ones. + */ +void check_remote_agent_compatibility(int agent_version, char *compatibility_str, size_t compatibility_str_max_size) { + elog(LOG, "Agent version=%d", agent_version); + if (agent_version != AGENT_PROTOCOL_VERSION) { char agent_version_str[1024]; @@ -279,6 +333,23 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str "consider to upgrade pg_probackup binary", agent_version_str, AGENT_PROTOCOL_VERSION_STR); } - assert(false); - elog(ERROR, " check_remote_agent_compatibility() not implemented"); + + if (strnlen(compatibility_str, compatibility_str_max_size) == compatibility_str_max_size) + { + elog(ERROR, "Corrupted remote compatibility protocol: compatibility string has no terminating \\0"); + } + + elog(LOG, "Agent compatibility params: '%s'", compatibility_str); + + /* checking compatibility params */ + { + char *buf[compatibility_str_max_size]; + + prepare_compatibility_str(buf, sizeof buf); + if(!strcmp(compatibility_str, buf)) + { + elog(ERROR, "Incompatible agent params, expected %s", buf); + } + } + } From 0604cce21de69a3e3b0ea6a6634a6dac53ae39ea Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 01:55:25 +0300 Subject: [PATCH 25/55] [PBCKP-236] draft, solution without macros cleanup --- src/utils/file.c | 4 +--- src/utils/remote.c | 48 +++++++++++++--------------------------------- 2 files changed, 14 insertions(+), 38 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index b0dc39ae..fa098394 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -281,7 +281,7 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > payload_buf_size) { - //TODO REVIEW XXX %zu is C99 but not ANSI S standard, should we cast to unsigned long? + //TODO REVIEW XXX %zu is C99 but not ANSI S compatible, should we use ints, %zu is also applied at data.c:501 data.c:1638?? elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size); } @@ -3330,8 +3330,6 @@ fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); - //TODO REVIEW XXX make INFO to LOG or VERBOSE - elog(INFO, "TODO REVIEW XXX sent agent compatibility\n %s", buf); break; } case FIO_STAT: /* Get information about file with specified path */ diff --git a/src/utils/remote.c b/src/utils/remote.c index af3e460c..2babe79a 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -277,41 +277,20 @@ static char* compatibility_params[] = { */ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { - char tmp_buf[1024]; - int size_inc = 0; - size_t result_size = 1; + size_t result_size = 0; size_t compatibility_params_array_size = sizeof compatibility_params / sizeof compatibility_params[0];; *compatibility_buf = '\0'; Assert(compatibility_params_array_size % 2 == 0); - //TODO !!!! for (int i = 0; i < compatibility_params_array_size; i+=2) { - size_inc = snprintf(compatibility_buf + size_inc, compatibility_buf_size, - "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, - compatibility_params[i], compatibility_params[i+1]); - -// size_inc = snprintf(tmp_buf, sizeof tmp_buf, -// "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, -// compatibility_params[i], compatibility_params[i+1]); - if (size_inc >= sizeof tmp_buf) - { - //TODO make Assert - elog(ERROR, "Compatibility params from agent doesn't fit to %zu chars, %s=%s", - sizeof tmp_buf - 1, compatibility_params[i], compatibility_params[i+1] ); - } - - result_size += size_inc; - if (result_size > compatibility_buf_size) - { - //TODO make Assert - elog(ERROR, "Can't fit compatibility string size %zu to buffer size %zu:\n%s\n%s", - result_size, compatibility_buf_size, compatibility_buf, tmp_buf); - } - strcat(compatibility_buf, tmp_buf); + result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, + "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, + compatibility_params[i], compatibility_params[i+1]); + Assert(result_size < compatibility_buf_size); } - return result_size; + return result_size + 1; } /* @@ -319,7 +298,7 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b */ void check_remote_agent_compatibility(int agent_version, char *compatibility_str, size_t compatibility_str_max_size) { - elog(LOG, "Agent version=%d", agent_version); + elog(LOG, "Agent version=%d\n", agent_version); if (agent_version != AGENT_PROTOCOL_VERSION) { @@ -331,25 +310,24 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str elog(ERROR, "Remote agent protocol version %s does not match local program protocol version %s, " "consider to upgrade pg_probackup binary", - agent_version_str, AGENT_PROTOCOL_VERSION_STR); + agent_version_str, AGENT_PROTOCOL_VERSION_STR); } + /* checking compatibility params */ if (strnlen(compatibility_str, compatibility_str_max_size) == compatibility_str_max_size) { elog(ERROR, "Corrupted remote compatibility protocol: compatibility string has no terminating \\0"); } - elog(LOG, "Agent compatibility params: '%s'", compatibility_str); + elog(LOG, "Agent compatibility params:\n%s", compatibility_str); - /* checking compatibility params */ { - char *buf[compatibility_str_max_size]; + char buf[compatibility_str_max_size]; prepare_compatibility_str(buf, sizeof buf); - if(!strcmp(compatibility_str, buf)) + if(strcmp(compatibility_str, buf)) { - elog(ERROR, "Incompatible agent params, expected %s", buf); + elog(ERROR, "Incompatible remote agent params, expected:\n%s", buf); } } - } From f61be78e783a5948750e5fc77f31eeab54a82ae2 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 04:39:18 +0300 Subject: [PATCH 26/55] [PBCKP-236] working solution cleaned up --- src/utils/remote.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 2babe79a..79456d9f 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -249,26 +249,13 @@ bool launch_agent(void) return true; } -#define COMPATIBILITY_VAL(macro) #macro, macro -#define COMPATIBILITY_STR(macro) #macro -#define COMPATIBILITY_VAL_STR(macro) #macro, COMPATIBILITY_STR(macro) +#define COMPATIBILITY_VAL_STR(macro) #macro, macro +#define COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) (snprintf(helper_buf, buf_size, "%d", macro), helper_buf) +#define COMPATIBILITY_VAL_INT(macro, helper_buf, buf_size) #macro, COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) #define COMPATIBILITY_VAL_SEPARATOR "=" #define COMPATIBILITY_LINE_SEPARATOR "\n" -static char* compatibility_params[] = { - COMPATIBILITY_VAL(PG_MAJORVERSION), - //TODO remove? - //TODO doesn't work macro name check for ints!!!! - COMPATIBILITY_VAL_STR(SIZEOF_VOID_P), - //TODO REVIEW XXX can use edition.h/extract_pgpro_edition() -#ifdef PGPRO_EDN - //TODO add vanilla - //TODO make "1c" -> "vanilla" - COMPATIBILITY_VAL(PGPRO_EDN), -#endif -}; - /* * Compose compatibility string to be sent by pg_probackup agent * through ssh and to be verified by pg_probackup peer. @@ -277,6 +264,18 @@ static char* compatibility_params[] = { */ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { + char compatibility_val_int_macro_helper_buf[32]; + char* compatibility_params[] = { + COMPATIBILITY_VAL_STR(PG_MAJORVERSION), +#ifdef PGPRO_EDN + //TODO REVIEW can use edition.h/extract_pgpro_edition() + COMPATIBILITY_VAL_STR(PGPRO_EDN), +#endif + //TODO REVIEW remove? no difference between 32/64 in global/pg_control. + COMPATIBILITY_VAL_INT(SIZEOF_VOID_P, + compatibility_val_int_macro_helper_buf, sizeof compatibility_val_int_macro_helper_buf), + }; + size_t result_size = 0; size_t compatibility_params_array_size = sizeof compatibility_params / sizeof compatibility_params[0];; From b2091cd2c277d28ec8bc4f1c7980b1c1798076ea Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 05:51:11 +0300 Subject: [PATCH 27/55] [PBCKP-236] [skip] removed unnecessary TODOs --- src/utils/file.c | 2 -- src/utils/remote.c | 16 ++++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index fa098394..5a2aa61a 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -268,7 +268,6 @@ fio_write_all(int fd, void const* buf, size_t size) return offs; } -//TODO REVIEW XXX move to remote.c???? /* Get version of remote agent */ void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) @@ -3217,7 +3216,6 @@ fio_delete_impl(mode_t mode, char *buf) } /* Execute commands at remote host */ -//TODO REVIEW XXX move to remote.c? void fio_communicate(int in, int out) { diff --git a/src/utils/remote.c b/src/utils/remote.c index 79456d9f..97c8f3d4 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -117,9 +117,6 @@ bool launch_agent(void) int infd[2]; int errfd[2]; int agent_version; - //TODO REVIEW XXX review buf_size - int payload_buf_size = 1024 * 8; - char payload_buf[payload_buf_size]; ssh_argc = 0; #ifdef WIN32 @@ -241,10 +238,13 @@ bool launch_agent(void) fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */ } - /* Make sure that remote agent has the same version, fork and other features to be binary compatible - */ - fio_get_agent_version(&agent_version, payload_buf, payload_buf_size); - check_remote_agent_compatibility(agent_version, payload_buf, payload_buf_size); + + /* Make sure that remote agent has the same version, fork and other features to be binary compatible */ + { + char payload_buf[1024]; + fio_get_agent_version(&agent_version, payload_buf, sizeof payload_buf); + check_remote_agent_compatibility(agent_version, payload_buf, sizeof payload_buf); + } return true; } @@ -268,7 +268,7 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b char* compatibility_params[] = { COMPATIBILITY_VAL_STR(PG_MAJORVERSION), #ifdef PGPRO_EDN - //TODO REVIEW can use edition.h/extract_pgpro_edition() + //TODO REVIEW can use edition.h/extract_pgpro_edition() or similar COMPATIBILITY_VAL_STR(PGPRO_EDN), #endif //TODO REVIEW remove? no difference between 32/64 in global/pg_control. From 56598848c260c1cae5a6fb9d739f840c105668ed Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 06:43:16 +0300 Subject: [PATCH 28/55] [PBCKP-236] ANSI C fix --- src/utils/remote.c | 2 +- tests/compatibility.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 97c8f3d4..a5294c70 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -321,7 +321,7 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str elog(LOG, "Agent compatibility params:\n%s", compatibility_str); { - char buf[compatibility_str_max_size]; + char buf[1024]; prepare_compatibility_str(buf, sizeof buf); if(strcmp(compatibility_str, buf)) diff --git a/tests/compatibility.py b/tests/compatibility.py index 0562b441..b0a9b0e5 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,7 +14,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 From 35df5060d5e1a20bdfa45dbaa5aab429f0fd4aa6 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 26 Sep 2022 06:00:03 +0300 Subject: [PATCH 29/55] [PBCKP-236] 1c+certified editions check --- src/utils/file.c | 1 - src/utils/remote.c | 58 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 5a2aa61a..242810b5 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -280,7 +280,6 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > payload_buf_size) { - //TODO REVIEW XXX %zu is C99 but not ANSI S compatible, should we use ints, %zu is also applied at data.c:501 data.c:1638?? elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size); } diff --git a/src/utils/remote.c b/src/utils/remote.c index a5294c70..786b4bfb 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -249,6 +249,57 @@ bool launch_agent(void) return true; } +/* PGPRO 10-13 check to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */ +static bool check_certified() +{ +#ifdef PGPRO_VERSION_STR + return strstr(PGPRO_VERSION_STR, "(certified)") || + strstr(PGPRO_VERSION_STR, ("(standard certified)")); +#endif + return false; +} + +//TODO REVIEW review coding standard https://jira.postgrespro.ru/browse/PBCKP-251 with @funny_falcon, newlines, braces etc +static char* extract_pg_edition_str() +{ + static char *vanilla = "vanilla"; + static char *std = "standard"; + static char *ent = "enterprise"; + static char *std_cert = "standard-certified"; + static char *ent_cert = "enterprise-certified"; + +#ifdef PGPRO_EDITION + if (strcasecmp(PGPRO_EDITION, "1C") == 0) + return vanilla; + + /* these "certified" checks are applicable to PGPRO from 9.6 up to 12 versions. + * 13+ certified versions are compatible to non-certified ones */ + if (PG_VERSION_NUM < 100000) + { + if (strcmp(PGPRO_EDITION, "standard-certified") == 0) + return std_cert; + else if (strcmp(PGPRO_EDITION, "enterprise-certified")) + return ent_cert; + else + Assert("Bad #define PGPRO_EDITION value" == 0); + } + + if (check_certified()) + { + if (strcmp(PGPRO_EDITION, "standard")) + return std_cert; + else if (strcmp(PGPRO_EDITION, "enterprise") == 0) + return ent_cert; + else + Assert("Bad #define PGPRO_EDITION value" == 0); + } + + return PGPRO_EDITION; +#else + return vanilla; +#endif +} + #define COMPATIBILITY_VAL_STR(macro) #macro, macro #define COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) (snprintf(helper_buf, buf_size, "%d", macro), helper_buf) #define COMPATIBILITY_VAL_INT(macro, helper_buf, buf_size) #macro, COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) @@ -267,11 +318,8 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b char compatibility_val_int_macro_helper_buf[32]; char* compatibility_params[] = { COMPATIBILITY_VAL_STR(PG_MAJORVERSION), -#ifdef PGPRO_EDN - //TODO REVIEW can use edition.h/extract_pgpro_edition() or similar - COMPATIBILITY_VAL_STR(PGPRO_EDN), -#endif - //TODO REVIEW remove? no difference between 32/64 in global/pg_control. + "edition", extract_pg_edition_str(), + /* 32/64 bits compatibility */ COMPATIBILITY_VAL_INT(SIZEOF_VOID_P, compatibility_val_int_macro_helper_buf, sizeof compatibility_val_int_macro_helper_buf), }; From 0dc826fc118ce5988b984b9ee7912a054920d941 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 26 Sep 2022 10:59:37 +0300 Subject: [PATCH 30/55] Fix packaging/Makefile.test typo [ci skip] --- packaging/Makefile.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/Makefile.test b/packaging/Makefile.test index f5e004f0..11c63619 100644 --- a/packaging/Makefile.test +++ b/packaging/Makefile.test @@ -130,10 +130,10 @@ build/test_suse: build/test_suse_15.1 build/test_suse_15.2 @echo Suse: done build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13 - @echo Rhel 15.1: done + @echo Suse 15.1: done build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 build/test_suse_15.2_14 - @echo Rhel 15.1: done + @echo Suse 15.2: done define test_suse docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ From 4d907c958d2cd74b6e85eaf426ed93b18e9f6c39 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 26 Sep 2022 11:01:21 +0300 Subject: [PATCH 31/55] Fix test packagin script for 9.6 [ci skip] --- packaging/test/scripts/rpm.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packaging/test/scripts/rpm.sh b/packaging/test/scripts/rpm.sh index 3b680699..87d430ef 100755 --- a/packaging/test/scripts/rpm.sh +++ b/packaging/test/scripts/rpm.sh @@ -77,6 +77,12 @@ if [ ${DISTRIB} == 'centos' ] && [ ${DISTRIB_VERSION} == '8' ]; then dnf -qy module disable postgresql fi +# PGDG doesn't support install of PG-9.6 from repo package anymore +if [ ${PG_VERSION} == '9.6' ] && [ ${DISTRIB_VERSION} == '7' ]; then + # ugly hack: use repo settings from PG10 + sed -i 's/10/9.6/' /etc/yum.repos.d/pgdg-redhat-all.repo +fi + yum install -y postgresql${PG_TOG}-server.x86_64 export PGDATA=/var/lib/pgsql/${PG_VERSION}/data From b3351b50d664b9a639537d4aea694d971c9cd7d5 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 27 Sep 2022 06:00:57 +0300 Subject: [PATCH 32/55] [PBCKP-236] final update --- src/utils/remote.c | 32 ++++++++++++-------------------- tests/compatibility.py | 8 ++++---- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 786b4bfb..7d86be4c 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -249,19 +249,18 @@ bool launch_agent(void) return true; } -/* PGPRO 10-13 check to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */ +#ifdef PGPRO_EDITION +/* PGPRO 10-13 checks to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */ static bool check_certified() { -#ifdef PGPRO_VERSION_STR return strstr(PGPRO_VERSION_STR, "(certified)") || strstr(PGPRO_VERSION_STR, ("(standard certified)")); -#endif - return false; } +#endif -//TODO REVIEW review coding standard https://jira.postgrespro.ru/browse/PBCKP-251 with @funny_falcon, newlines, braces etc static char* extract_pg_edition_str() { + static char *_1C = "1C"; static char *vanilla = "vanilla"; static char *std = "standard"; static char *ent = "enterprise"; @@ -269,26 +268,19 @@ static char* extract_pg_edition_str() static char *ent_cert = "enterprise-certified"; #ifdef PGPRO_EDITION - if (strcasecmp(PGPRO_EDITION, "1C") == 0) + if (strcmp(PGPRO_EDITION, _1C) == 0) return vanilla; - /* these "certified" checks are applicable to PGPRO from 9.6 up to 12 versions. - * 13+ certified versions are compatible to non-certified ones */ if (PG_VERSION_NUM < 100000) - { - if (strcmp(PGPRO_EDITION, "standard-certified") == 0) - return std_cert; - else if (strcmp(PGPRO_EDITION, "enterprise-certified")) - return ent_cert; - else - Assert("Bad #define PGPRO_EDITION value" == 0); - } + return PGPRO_EDITION; - if (check_certified()) + /* these "certified" checks are applicable to PGPRO from 10 up to 12 versions. + * 13+ certified versions are compatible to non-certified ones */ + if (PG_VERSION_NUM < 130000 && check_certified()) { - if (strcmp(PGPRO_EDITION, "standard")) + if (strcmp(PGPRO_EDITION, std) == 0) return std_cert; - else if (strcmp(PGPRO_EDITION, "enterprise") == 0) + else if (strcmp(PGPRO_EDITION, ent) == 0) return ent_cert; else Assert("Bad #define PGPRO_EDITION value" == 0); @@ -374,7 +366,7 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str prepare_compatibility_str(buf, sizeof buf); if(strcmp(compatibility_str, buf)) { - elog(ERROR, "Incompatible remote agent params, expected:\n%s", buf); + elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s ", buf, compatibility_str); } } } diff --git a/tests/compatibility.py b/tests/compatibility.py index b0a9b0e5..04af1478 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,7 +14,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 @@ -27,13 +27,13 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): calling probackup PGPROEE9_6 agent from PGPROEE11 probackup master for DELTA backup causes the PBCKP-236 problem - please correct path for agent's pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + please correct path for agent's pg_path_remote_version = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' """ self.verbose = True self.remote = True # please use your own local path - pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + pg_path_remote_version = '/home/avaness/postgres/postgres.build.clean/bin' src_pg = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'src'), @@ -70,7 +70,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): source_pgdata = src_pg.data_dir, destination_node = dst_pg, # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version - options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_ee_9_6] + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_remote_version] ) # Clean after yourself From 6e671232b8791ccb8bf6090c9dcf38081fc9c2dd Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 28 Sep 2022 04:32:47 +0300 Subject: [PATCH 33/55] [PBCKP-236] final update after review --- src/utils/remote.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 7d86be4c..0f254d14 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -292,12 +292,8 @@ static char* extract_pg_edition_str() #endif } -#define COMPATIBILITY_VAL_STR(macro) #macro, macro -#define COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) (snprintf(helper_buf, buf_size, "%d", macro), helper_buf) -#define COMPATIBILITY_VAL_INT(macro, helper_buf, buf_size) #macro, COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) - -#define COMPATIBILITY_VAL_SEPARATOR "=" -#define COMPATIBILITY_LINE_SEPARATOR "\n" +#define COMPATIBILITY_VAL_STR(macro) { #macro, macro, 0 } +#define COMPATIBILITY_VAL_INT(macro) { #macro, NULL, macro } /* * Compose compatibility string to be sent by pg_probackup agent @@ -307,13 +303,10 @@ static char* extract_pg_edition_str() */ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { - char compatibility_val_int_macro_helper_buf[32]; - char* compatibility_params[] = { + struct { const char* name; const char* strval; int intval; } compatibility_params[] = { COMPATIBILITY_VAL_STR(PG_MAJORVERSION), - "edition", extract_pg_edition_str(), - /* 32/64 bits compatibility */ - COMPATIBILITY_VAL_INT(SIZEOF_VOID_P, - compatibility_val_int_macro_helper_buf, sizeof compatibility_val_int_macro_helper_buf), + { "edition", extract_pg_edition_str(), 0 }, + COMPATIBILITY_VAL_INT(SIZEOF_VOID_P), }; size_t result_size = 0; @@ -324,9 +317,16 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b for (int i = 0; i < compatibility_params_array_size; i+=2) { - result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, - "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, - compatibility_params[i], compatibility_params[i+1]); + if (compatibility_params[i].strval != NULL) + result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, + "%s=%s/n", + compatibility_params[i].name, + compatibility_params[i].strval); + else + result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, + "%s=%d/n", + compatibility_params[i].name, + compatibility_params[i].intval); Assert(result_size < compatibility_buf_size); } return result_size + 1; @@ -349,7 +349,7 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str elog(ERROR, "Remote agent protocol version %s does not match local program protocol version %s, " "consider to upgrade pg_probackup binary", - agent_version_str, AGENT_PROTOCOL_VERSION_STR); + agent_version_str, AGENT_PROTOCOL_VERSION_STR); } /* checking compatibility params */ From 1ce38ed70cccabcdea5dbda7f1030424ceeeef03 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 28 Sep 2022 04:52:11 +0300 Subject: [PATCH 34/55] [PBCKP-236] final update after review --- src/utils/remote.c | 7 ++----- tests/compatibility.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 0f254d14..f3608e56 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -310,12 +310,9 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b }; size_t result_size = 0; - size_t compatibility_params_array_size = sizeof compatibility_params / sizeof compatibility_params[0];; - *compatibility_buf = '\0'; - Assert(compatibility_params_array_size % 2 == 0); - for (int i = 0; i < compatibility_params_array_size; i+=2) + for (int i = 0; i < sizeof compatibility_params; i+=2) { if (compatibility_params[i].strval != NULL) result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, @@ -366,7 +363,7 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str prepare_compatibility_str(buf, sizeof buf); if(strcmp(compatibility_str, buf)) { - elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s ", buf, compatibility_str); + elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s", buf, compatibility_str); } } } diff --git a/tests/compatibility.py b/tests/compatibility.py index 04af1478..4e5e27f0 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,7 +14,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 From c52659791b91012b13a2a8ebec1367dddc60187b Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 29 Sep 2022 03:01:57 +0300 Subject: [PATCH 35/55] [PBCKP-236] assert fix --- src/utils/remote.c | 17 +++++++++++++---- tests/compatibility.py | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index f3608e56..91468b54 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -295,6 +295,9 @@ static char* extract_pg_edition_str() #define COMPATIBILITY_VAL_STR(macro) { #macro, macro, 0 } #define COMPATIBILITY_VAL_INT(macro) { #macro, NULL, macro } +#define COMPATIBILITY_VAL_SEPARATOR "=" +#define COMPATIBILITY_LINE_SEPARATOR "\n" + /* * Compose compatibility string to be sent by pg_probackup agent * through ssh and to be verified by pg_probackup peer. @@ -303,7 +306,13 @@ static char* extract_pg_edition_str() */ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { - struct { const char* name; const char* strval; int intval; } compatibility_params[] = { + typedef struct compatibility_param_tag { + const char* name; + const char* strval; + int intval; + } compatibility_param; + + compatibility_param compatibility_params[] = { COMPATIBILITY_VAL_STR(PG_MAJORVERSION), { "edition", extract_pg_edition_str(), 0 }, COMPATIBILITY_VAL_INT(SIZEOF_VOID_P), @@ -312,16 +321,16 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b size_t result_size = 0; *compatibility_buf = '\0'; - for (int i = 0; i < sizeof compatibility_params; i+=2) + for (int i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) { if (compatibility_params[i].strval != NULL) result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, - "%s=%s/n", + "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, compatibility_params[i].name, compatibility_params[i].strval); else result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, - "%s=%d/n", + "%s" COMPATIBILITY_VAL_SEPARATOR "%d" COMPATIBILITY_LINE_SEPARATOR, compatibility_params[i].name, compatibility_params[i].intval); Assert(result_size < compatibility_buf_size); diff --git a/tests/compatibility.py b/tests/compatibility.py index 4e5e27f0..04af1478 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,7 +14,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 From 03d55d079b836d285716a3df67a213fa1674a50a Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 29 Sep 2022 05:03:51 +0300 Subject: [PATCH 36/55] [PBCKP-236] fix excessive warnings for vanilla --- src/utils/remote.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 91468b54..9feb44a9 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -260,14 +260,14 @@ static bool check_certified() static char* extract_pg_edition_str() { - static char *_1C = "1C"; static char *vanilla = "vanilla"; +#ifdef PGPRO_EDITION + static char *_1C = "1C"; static char *std = "standard"; static char *ent = "enterprise"; static char *std_cert = "standard-certified"; static char *ent_cert = "enterprise-certified"; -#ifdef PGPRO_EDITION if (strcmp(PGPRO_EDITION, _1C) == 0) return vanilla; From 23d5ee4abfb57506fe3f1400b4e9c635a42e6bf6 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Tue, 11 Oct 2022 12:20:41 +0300 Subject: [PATCH 37/55] [PBCKP-235] review fixes tags: pg_probackup --- .travis.yml | 1 - README.md | 32 ++++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index f113d05c..8a67e77b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,6 @@ env: jobs: allow_failures: - if: env(PG_BRANCH) = master - - if: env(PG_BRANCH) = REL_15_STABLE - if: env(PG_BRANCH) = REL9_5_STABLE # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) diff --git a/README.md b/README.md index 43397847..d1ccd986 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `PTRACK` backup support provided via following options: * vanilla PostgreSQL 11, 12, 13, 14, 15 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13, 14 -* Postgres Pro Enterprise 11, 12, 13, 14 +* Postgres Pro Standard 11, 12, 13, 14, 15 +* Postgres Pro Enterprise 11, 12, 13, 14, 15 ## Limitations @@ -137,8 +137,8 @@ sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{15,14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{15,14,13,12,11,10,9.6}-dbg #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' @@ -148,35 +148,35 @@ sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). From d808a16640be611e363b66900b18b2b6f8a52747 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 11 Oct 2022 16:09:57 +0300 Subject: [PATCH 38/55] [PBCKP-236] removed excessive brackets --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 9feb44a9..addd73dc 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -254,7 +254,7 @@ bool launch_agent(void) static bool check_certified() { return strstr(PGPRO_VERSION_STR, "(certified)") || - strstr(PGPRO_VERSION_STR, ("(standard certified)")); + strstr(PGPRO_VERSION_STR, "(standard certified)"); } #endif From 96ad6e2eb9d0a9a6ef166d61d2bffa4098354cd7 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 12 Oct 2022 17:11:58 +0300 Subject: [PATCH 39/55] version macro increment, Release 2.5.9 --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 533b05d5..27deeee9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -344,7 +344,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.8" +#define PROGRAM_VERSION "2.5.9" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 From 80efb85029a254c0a885931058d08c260c3c35d6 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 30 Sep 2022 11:06:16 +0300 Subject: [PATCH 40/55] [PBCKP-146] Small fix for remote_agent --- src/pg_probackup.c | 3 ++- src/pg_probackup.h | 2 +- src/utils/configuration.c | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1f6b6313..84968527 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -88,7 +88,7 @@ bool perm_slot = false; /* backup options */ bool backup_logs = false; bool smooth_checkpoint; -char *remote_agent; +bool remote_agent = false; static char *backup_note = NULL; /* catchup options */ static char *catchup_source_pgdata = NULL; @@ -361,6 +361,7 @@ main(int argc, char *argv[]) elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' " "is launched as an agent for pg_probackup binary with version '%s'", PROGRAM_VERSION, argv[2]); + remote_agent = true; fio_communicate(STDIN_FILENO, STDOUT_FILENO); return 0; case HELP_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 533b05d5..495fbdca 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -798,7 +798,7 @@ extern bool perm_slot; extern bool smooth_checkpoint; /* remote probackup options */ -extern char* remote_agent; +extern bool remote_agent; extern bool exclusive_backup; diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 98c3b299..93f29c48 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -531,7 +531,6 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], opt = option_find(c, options); if (opt - && !remote_agent && opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT) elog(ERROR, "Option %s cannot be specified in command line", opt->lname); From 4730857b7946e0fb136e66acb27de44ca08e4977 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 30 Sep 2022 12:22:14 +0300 Subject: [PATCH 41/55] [PBCKP-146] - fio_get_crc32 - add "missing_ok" parameter --- src/archive.c | 9 ++++----- src/data.c | 9 ++++++--- src/utils/file.c | 22 +++++++++++++++------- src/utils/file.h | 3 ++- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/archive.c b/src/archive.c index 1a19c3d8..2ae86bd6 100644 --- a/src/archive.c +++ b/src/archive.c @@ -512,8 +512,8 @@ part_opened: pg_crc32 crc32_src; pg_crc32 crc32_dst; - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false); + crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); + crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false, false); if (crc32_src == crc32_dst) { @@ -760,9 +760,8 @@ part_opened: pg_crc32 crc32_src; pg_crc32 crc32_dst; - /* TODO: what if one of them goes missing? */ - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true); + crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); + crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true, false); if (crc32_src == crc32_dst) { diff --git a/src/data.c b/src/data.c index 5c5fdf4f..753f247f 100644 --- a/src/data.c +++ b/src/data.c @@ -801,8 +801,11 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, (prev_file && file->exists_in_prev && file->mtime <= parent_backup_time)) { - - file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); + /* + * file could be deleted under our feets. + * But then backup_non_data_file_internal will handle it safely + */ + file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true); /* ...and checksum is the same... */ if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) @@ -1327,7 +1330,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (already_exists) { /* compare checksums of already existing file and backup file */ - pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false); + pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false); if (file_crc == tmp_file->crc) { diff --git a/src/utils/file.c b/src/utils/file.c index 7103c8f1..727b48c6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1355,9 +1355,15 @@ fio_sync(char const* path, fio_location location) } } +enum { + GET_CRC32_DECOMPRESS = 1, + GET_CRC32_MISSING_OK = 2 +}; + /* Get crc32 of file */ pg_crc32 -fio_get_crc32(const char *file_path, fio_location location, bool decompress) +fio_get_crc32(const char *file_path, fio_location location, + bool decompress, bool missing_ok) { if (fio_is_remote(location)) { @@ -1370,7 +1376,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress) hdr.arg = 0; if (decompress) - hdr.arg = 1; + hdr.arg = GET_CRC32_DECOMPRESS; + if (missing_ok) + hdr.arg |= GET_CRC32_MISSING_OK; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len); @@ -1381,9 +1389,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress) else { if (decompress) - return pgFileGetCRCgz(file_path, true, true); + return pgFileGetCRCgz(file_path, true, missing_ok); else - return pgFileGetCRC(file_path, true, true); + return pgFileGetCRC(file_path, true, missing_ok); } } @@ -3380,10 +3388,10 @@ fio_communicate(int in, int out) break; case FIO_GET_CRC32: /* calculate crc32 for a file */ - if (hdr.arg == 1) - crc = pgFileGetCRCgz(buf, true, true); + if ((hdr.arg & GET_CRC32_DECOMPRESS)) + crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); else - crc = pgFileGetCRC(buf, true, true); + crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; case FIO_GET_CHECKSUM_MAP: diff --git a/src/utils/file.h b/src/utils/file.h index a554b4ab..ec478b45 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -120,7 +120,8 @@ extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); extern void fio_disconnect(void); extern int fio_sync(char const* path, fio_location location); -extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress); +extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, + bool decompress, bool missing_ok); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); From e16c62e8fd705d65d2cc1a7fd3beb0ea40e31277 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 5 Oct 2022 12:16:10 +0300 Subject: [PATCH 42/55] [PBCKP-146] prettify forkname handling. --- src/catalog.c | 5 ++- src/dir.c | 89 ++++++++++++++++++++++------------------------ src/pg_probackup.h | 2 ++ 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 47513096..03099d1a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1132,6 +1132,9 @@ get_backup_filelist(pgBackup *backup, bool strict) if (get_control_value_int64(buf, "hdr_size", &hdr_size, false)) file->hdr_size = (int) hdr_size; + if (file->external_dir_num == 0) + set_forkname(file); + parray_append(files, file); } @@ -2488,7 +2491,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, char control_path[MAXPGPATH]; char control_path_temp[MAXPGPATH]; size_t i = 0; - #define BUFFERSZ 1024*1024 + #define BUFFERSZ (1024*1024) char *buf; int64 backup_size_on_disk = 0; int64 uncompressed_size_on_disk = 0; diff --git a/src/dir.c b/src/dir.c index 561586f8..00e918d0 100644 --- a/src/dir.c +++ b/src/dir.c @@ -758,57 +758,22 @@ dir_check_file(pgFile *file, bool backup_logs) return CHECK_FALSE; else if (isdigit(file->name[0])) { - char *fork_name; - int len; - char suffix[MAXPGPATH]; + set_forkname(file); - fork_name = strstr(file->name, "_"); - if (fork_name) + if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */ + return CHECK_FALSE; + else if (file->forkName != none) + return CHECK_TRUE; + + /* Set is_datafile flag */ { - /* Auxiliary fork of the relfile */ - if (strcmp(fork_name, "_vm") == 0) - file->forkName = vm; - - else if (strcmp(fork_name, "_fsm") == 0) - file->forkName = fsm; - - else if (strcmp(fork_name, "_cfm") == 0) - file->forkName = cfm; - - else if (strcmp(fork_name, "_ptrack") == 0) - file->forkName = ptrack; - - else if (strcmp(fork_name, "_init") == 0) - file->forkName = init; - - // extract relOid for certain forks - if (file->forkName == vm || - file->forkName == fsm || - file->forkName == init || - file->forkName == cfm) - { - // sanity - if (sscanf(file->name, "%u_*", &(file->relOid)) != 1) - file->relOid = 0; - } - - /* Do not backup ptrack files */ - if (file->forkName == ptrack) - return CHECK_FALSE; - } - else - { - - len = strlen(file->name); - /* reloid.cfm */ - if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) - return CHECK_TRUE; + char suffix[MAXFNAMELEN]; + /* check if file is datafile */ sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), &(file->segno), suffix); - if (sscanf_res == 0) - elog(ERROR, "Cannot parse file name \"%s\"", file->name); - else if (sscanf_res == 1 || sscanf_res == 2) + Assert(sscanf_res > 0); /* since first char is digit */ + if (sscanf_res == 1 || sscanf_res == 2) file->is_datafile = true; } } @@ -1954,3 +1919,35 @@ pfilearray_clear_locks(parray *file_list) pg_atomic_clear_flag(&file->lock); } } + +/* Set forkName if possible */ +void +set_forkname(pgFile *file) +{ + int name_len = strlen(file->name); + + /* Auxiliary fork of the relfile */ + if (name_len > 3 && strcmp(file->name + name_len - 3, "_vm") == 0) + file->forkName = vm; + + else if (name_len > 4 && strcmp(file->name + name_len - 4, "_fsm") == 0) + file->forkName = fsm; + + else if (name_len > 4 && strcmp(file->name + name_len - 4, ".cfm") == 0) + file->forkName = cfm; + + else if (name_len > 5 && strcmp(file->name + name_len - 5, "_init") == 0) + file->forkName = init; + + else if (name_len > 7 && strcmp(file->name + name_len - 7, "_ptrack") == 0) + file->forkName = ptrack; + + // extract relOid for certain forks + + if ((file->forkName == vm || + file->forkName == fsm || + file->forkName == init || + file->forkName == cfm) && + (sscanf(file->name, "%u*", &(file->relOid)) != 1)) + file->relOid = 0; +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 495fbdca..bc9f9b8a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -215,6 +215,7 @@ typedef enum CompressAlg typedef enum ForkName { + none, vm, fsm, cfm, @@ -1091,6 +1092,7 @@ extern int pgCompareString(const void *str1, const void *str2); extern int pgPrefixCompareString(const void *str1, const void *str2); extern int pgCompareOid(const void *f1, const void *f2); extern void pfilearray_clear_locks(parray *file_list); +extern void set_forkname(pgFile *file); /* in data.c */ extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, From 8f504fc95cbace9297da363ac098126b4e75c6c8 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 10 Oct 2022 17:07:41 +0300 Subject: [PATCH 43/55] [PBCKP-146] stabilize couple of tests. --- tests/cfs_backup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index d820360f..436db31e 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -995,6 +995,11 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): "FROM generate_series(0,256) i".format('t1', tblspace_name) ) + self.node.safe_psql( + "postgres", + "CHECKPOINT" + ) + list_cmf = find_by_extensions( [self.get_tblspace_path(self.node, tblspace_name)], ['.cfm']) @@ -1044,6 +1049,11 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): "FROM generate_series(0,256) i".format('t1', tblspace_name) ) + self.node.safe_psql( + "postgres", + "CHECKPOINT" + ) + list_data_files = find_by_pattern( [self.get_tblspace_path(self.node, tblspace_name)], '^.*/\d+$') From 51a141c4b0b07899c001f75c5c172492cbefa076 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 11 Oct 2022 18:59:48 +0300 Subject: [PATCH 44/55] [PBCKP-146] fix cfs test python3 compatibility --- tests/cfs_restore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 07cf891a..611afc49 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -103,6 +103,7 @@ class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) ) + tblspace = str(tblspace) self.assertTrue( tblspace_name in tblspace and "compression=true" in tblspace, "ERROR: The tablespace not restored or it restored without compressions" From 3e17c8c8daa21ba8d79fecc614c591c429dda7d1 Mon Sep 17 00:00:00 2001 From: "Andrew A. Bille" Date: Mon, 19 Sep 2022 12:25:23 +0700 Subject: [PATCH 45/55] Fix remembered check gdb flag in test with GDB --- tests/locking.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/locking.py b/tests/locking.py index 0fe954ca..4042a146 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -419,6 +419,8 @@ class LockingTest(ProbackupTest, unittest.TestCase): and stop it in the middle, take page backup. Expect PAGE backup to be successfully executed """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), From 26f9992b2ab484a0bdd605d9669b2c86a07c28ed Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 24 Oct 2022 18:28:30 +0300 Subject: [PATCH 46/55] [PBCKP-236] added PGPROBACKUP_MANUAL testing and PGPROBACKUP_SSH_AGENT_PATH flags. --- tests/compatibility.py | 53 +++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 04af1478..6c2bc920 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -8,32 +8,48 @@ import shutil module_name = 'compatibility' +def check_manual_tests_enabled(): + return 'PGPROBACKUP_MANUAL' in os.environ and os.environ['PGPROBACKUP_MANUAL'] == 'ON' + + +def check_ssh_agent_path_exists(): + return 'PGPROBACKUP_SSH_AGENT_PATH' in os.environ + + class CompatibilityTest(ProbackupTest, unittest.TestCase): def setUp(self): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - @unittest.skip("skip") + @unittest.skipUnless(check_manual_tests_enabled(), 'skip manual test') + @unittest.skipUnless(check_ssh_agent_path_exists(), 'skip no ssh agent path exist') + # @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 - This test requires builds both PGPROEE11 and PGPROEE9_6 + This test exposures ticket error using pg_probackup builds for both PGPROEE11 and PGPROEE9_6 - prerequisites: - - git tag for PBCKP 2.5.1 - - master probackup build should be inside PGPROEE11 - - agent probackup build is inside PGPROEE9_6 + Prerequisites: + - pg_probackup git tag for PBCKP 2.5.1 + - master pg_probackup build should be made for PGPROEE11 + - agent pg_probackup build should be made for PGPROEE9_6 - calling probackup PGPROEE9_6 agent from PGPROEE11 probackup master for DELTA backup causes the PBCKP-236 problem + Calling probackup PGPROEE9_6 pg_probackup agent from PGPROEE11 pg_probackup master for DELTA backup causes + the PBCKP-236 problem - please correct path for agent's pg_path_remote_version = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + Please give env variables PROBACKUP_MANUAL=ON;PGPROBACKUP_SSH_AGENT_PATH= + for the test + + Please make path for agent's pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + without pg_probackup executable """ self.verbose = True self.remote = True - # please use your own local path - pg_path_remote_version = '/home/avaness/postgres/postgres.build.clean/bin' + # please use your own local path like + # pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.clean/bin/' + pgprobackup_ssh_agent_path = os.environ['PGPROBACKUP_SSH_AGENT_PATH'] src_pg = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'src'), @@ -47,14 +63,13 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): # do full catchup dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) self.catchup_node( - backup_mode = 'FULL', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, + backup_mode='FULL', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, options=['-d', 'postgres', '-p', str(src_pg.port), '--stream'] ) - dst_options = {} - dst_options['port'] = str(dst_pg.port) + dst_options = {'port': str(dst_pg.port)} self.set_auto_conf(dst_pg, dst_options) dst_pg.slow_start() dst_pg.stop() @@ -66,11 +81,11 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): # do delta catchup with remote pg_probackup agent with another postgres major version # this DELTA backup should fail without PBCKP-236 patch. self.catchup_node( - backup_mode = 'DELTA', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, + backup_mode='DELTA', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version - options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_remote_version] + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pgprobackup_ssh_agent_path] ) # Clean after yourself From 97355f1562041136c952add40b5ccafc76eaff58 Mon Sep 17 00:00:00 2001 From: dlepikhova <43872363+dlepikhova@users.noreply.github.com> Date: Wed, 26 Oct 2022 18:34:36 +0500 Subject: [PATCH 47/55] release_2_5_9-pbckp-227 (#533) [PBCKP-227]: Fix some potential problems in pg_probackup code * Fix fwrite parameters in fio_open_stream * Remove unused get_system_dbstate function * Set actual sourse for fields in instance_options to SOURCE_DEFAULT * Remove get_system_dbstate declaration from header file --- src/configure.c | 132 ++++++++++++++++++++++----------------------- src/pg_probackup.h | 1 - src/util.c | 16 ------ src/utils/file.c | 2 +- 4 files changed, 67 insertions(+), 84 deletions(-) diff --git a/src/configure.c b/src/configure.c index 6e8700de..f7befb0c 100644 --- a/src/configure.c +++ b/src/configure.c @@ -53,7 +53,7 @@ ConfigOption instance_options[] = /* Instance options */ { 's', 'D', "pgdata", - &instance_config.pgdata, SOURCE_CMD, 0, + &instance_config.pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, { @@ -70,49 +70,49 @@ ConfigOption instance_options[] = #endif { 's', 'E', "external-dirs", - &instance_config.external_dir_str, SOURCE_CMD, 0, + &instance_config.external_dir_str, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Connection options */ { 's', 'd', "pgdatabase", - &instance_config.conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance_config.conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'h', "pghost", - &instance_config.conn_opt.pghost, SOURCE_CMD, 0, + &instance_config.conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'p', "pgport", - &instance_config.conn_opt.pgport, SOURCE_CMD, 0, + &instance_config.conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'U', "pguser", - &instance_config.conn_opt.pguser, SOURCE_CMD, 0, + &instance_config.conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, /* Replica options */ { 's', 202, "master-db", - &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 203, "master-host", - &instance_config.master_conn_opt.pghost, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 204, "master-port", - &instance_config.master_conn_opt.pgport, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 205, "master-user", - &instance_config.master_conn_opt.pguser, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { @@ -128,17 +128,17 @@ ConfigOption instance_options[] = }, { 's', 208, "archive-host", - &instance_config.archive.host, SOURCE_CMD, 0, + &instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 209, "archive-port", - &instance_config.archive.port, SOURCE_CMD, 0, + &instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 210, "archive-user", - &instance_config.archive.user, SOURCE_CMD, 0, + &instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { @@ -149,37 +149,37 @@ ConfigOption instance_options[] = /* Logging options */ { 'f', 212, "log-level-console", - assign_log_level_console, SOURCE_CMD, 0, + assign_log_level_console, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_level_console }, { 'f', 213, "log-level-file", - assign_log_level_file, SOURCE_CMD, 0, + assign_log_level_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_level_file }, { 'f', 214, "log-format-console", - assign_log_format_console, SOURCE_CMD_STRICT, 0, + assign_log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_format_console }, { 'f', 215, "log-format-file", - assign_log_format_file, SOURCE_CMD, 0, + assign_log_format_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_format_file }, { 's', 216, "log-filename", - &instance_config.logger.log_filename, SOURCE_CMD, 0, + &instance_config.logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 217, "error-log-filename", - &instance_config.logger.error_log_filename, SOURCE_CMD, 0, + &instance_config.logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 218, "log-directory", - &instance_config.logger.log_directory, SOURCE_CMD, 0, + &instance_config.logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { @@ -195,64 +195,64 @@ ConfigOption instance_options[] = /* Retention options */ { 'u', 221, "retention-redundancy", - &instance_config.retention_redundancy, SOURCE_CMD, 0, + &instance_config.retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { 'u', 222, "retention-window", - &instance_config.retention_window, SOURCE_CMD, 0, + &instance_config.retention_window, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { 'u', 223, "wal-depth", - &instance_config.wal_depth, SOURCE_CMD, 0, + &instance_config.wal_depth, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { 'f', 224, "compress-algorithm", - assign_compress_alg, SOURCE_CMD, 0, + assign_compress_alg, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, get_compress_alg }, { 'u', 225, "compress-level", - &instance_config.compress_level, SOURCE_CMD, 0, + &instance_config.compress_level, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { 's', 226, "remote-proto", - &instance_config.remote.proto, SOURCE_CMD, 0, + &instance_config.remote.proto, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 227, "remote-host", - &instance_config.remote.host, SOURCE_CMD, 0, + &instance_config.remote.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 228, "remote-port", - &instance_config.remote.port, SOURCE_CMD, 0, + &instance_config.remote.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 229, "remote-path", - &instance_config.remote.path, SOURCE_CMD, 0, + &instance_config.remote.path, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 230, "remote-user", - &instance_config.remote.user, SOURCE_CMD, 0, + &instance_config.remote.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 231, "ssh-options", - &instance_config.remote.ssh_options, SOURCE_CMD, 0, + &instance_config.remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 232, "ssh-config", - &instance_config.remote.ssh_config, SOURCE_CMD, 0, + &instance_config.remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 0 } @@ -412,7 +412,7 @@ readInstanceConfigFile(InstanceState *instanceState) /* Instance options */ { 's', 'D', "pgdata", - &instance->pgdata, SOURCE_CMD, 0, + &instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, { @@ -429,49 +429,49 @@ readInstanceConfigFile(InstanceState *instanceState) #endif { 's', 'E', "external-dirs", - &instance->external_dir_str, SOURCE_CMD, 0, + &instance->external_dir_str, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Connection options */ { 's', 'd', "pgdatabase", - &instance->conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance->conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'h', "pghost", - &instance->conn_opt.pghost, SOURCE_CMD, 0, + &instance->conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'p', "pgport", - &instance->conn_opt.pgport, SOURCE_CMD, 0, + &instance->conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'U', "pguser", - &instance->conn_opt.pguser, SOURCE_CMD, 0, + &instance->conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, /* Replica options */ { 's', 202, "master-db", - &instance->master_conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance->master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 203, "master-host", - &instance->master_conn_opt.pghost, SOURCE_CMD, 0, + &instance->master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 204, "master-port", - &instance->master_conn_opt.pgport, SOURCE_CMD, 0, + &instance->master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 205, "master-user", - &instance->master_conn_opt.pguser, SOURCE_CMD, 0, + &instance->master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { @@ -487,66 +487,66 @@ readInstanceConfigFile(InstanceState *instanceState) }, { 's', 208, "archive-host", - &instance_config.archive.host, SOURCE_CMD, 0, + &instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 209, "archive-port", - &instance_config.archive.port, SOURCE_CMD, 0, + &instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 210, "archive-user", - &instance_config.archive.user, SOURCE_CMD, 0, + &instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 211, "restore-command", - &instance->restore_command, SOURCE_CMD, 0, + &instance->restore_command, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, /* Instance options */ { 's', 'D', "pgdata", - &instance->pgdata, SOURCE_CMD, 0, + &instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Logging options */ { 's', 212, "log-level-console", - &log_level_console, SOURCE_CMD, 0, + &log_level_console, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 213, "log-level-file", - &log_level_file, SOURCE_CMD, 0, + &log_level_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 214, "log-format-console", - &log_format_console, SOURCE_CMD_STRICT, 0, + &log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 215, "log-format-file", - &log_format_file, SOURCE_CMD, 0, + &log_format_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 216, "log-filename", - &instance->logger.log_filename, SOURCE_CMD, 0, + &instance->logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 217, "error-log-filename", - &instance->logger.error_log_filename, SOURCE_CMD, 0, + &instance->logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 218, "log-directory", - &instance->logger.log_directory, SOURCE_CMD, 0, + &instance->logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { @@ -562,64 +562,64 @@ readInstanceConfigFile(InstanceState *instanceState) /* Retention options */ { 'u', 221, "retention-redundancy", - &instance->retention_redundancy, SOURCE_CMD, 0, + &instance->retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { 'u', 222, "retention-window", - &instance->retention_window, SOURCE_CMD, 0, + &instance->retention_window, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { 'u', 223, "wal-depth", - &instance->wal_depth, SOURCE_CMD, 0, + &instance->wal_depth, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { 's', 224, "compress-algorithm", - &compress_alg, SOURCE_CMD, 0, + &compress_alg, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 'u', 225, "compress-level", - &instance->compress_level, SOURCE_CMD, 0, + &instance->compress_level, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { 's', 226, "remote-proto", - &instance->remote.proto, SOURCE_CMD, 0, + &instance->remote.proto, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 227, "remote-host", - &instance->remote.host, SOURCE_CMD, 0, + &instance->remote.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 228, "remote-port", - &instance->remote.port, SOURCE_CMD, 0, + &instance->remote.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 229, "remote-path", - &instance->remote.path, SOURCE_CMD, 0, + &instance->remote.path, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 230, "remote-user", - &instance->remote.user, SOURCE_CMD, 0, + &instance->remote.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 231, "ssh-options", - &instance->remote.ssh_options, SOURCE_CMD, 0, + &instance->remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 232, "ssh-config", - &instance->remote.ssh_config, SOURCE_CMD, 0, + &instance->remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 0 } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bc9f9b8a..ffb74da1 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1175,7 +1175,6 @@ extern uint64 get_system_identifier(const char *pgdata_path, fio_location locati extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); -extern DBState get_system_dbstate(const char *pgdata_path, fio_location location); extern uint32 get_xlog_seg_size(const char *pgdata_path); extern void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo); extern void set_min_recovery_point(pgFile *file, const char *backup_path, diff --git a/src/util.c b/src/util.c index 4d6c50a0..d19877f0 100644 --- a/src/util.c +++ b/src/util.c @@ -349,22 +349,6 @@ get_pgcontrol_checksum(const char *pgdata_path) return ControlFile.crc; } -DBState -get_system_dbstate(const char *pgdata_path, fio_location location) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, location); - if (buffer == NULL) - return 0; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.state; -} - void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo) { diff --git a/src/utils/file.c b/src/utils/file.c index 727b48c6..e32696f1 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -305,7 +305,7 @@ fio_open_stream(char const* path, fio_location location) IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); #ifdef WIN32 f = tmpfile(); - IO_CHECK(fwrite(f, 1, hdr.size, fio_stdin_buffer), hdr.size); + IO_CHECK(fwrite(fio_stdin_buffer, 1, hdr.size, f), hdr.size); SYS_CHECK(fseek(f, 0, SEEK_SET)); #else f = fmemopen(fio_stdin_buffer, hdr.size, "r"); From 4ac1b536a5f6be93901ad2f7fef216d80d868229 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Fri, 14 Oct 2022 14:01:41 +0300 Subject: [PATCH 48/55] [PBCKP-235] README.md now contains correct package versions tags: pg_probackup --- .travis.yml | 5 ++-- README.md | 72 ++++++++++++++++++++++++++--------------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d0d786c..bd3c8a09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,8 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master + - PG_VERSION=16 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=REL_15_STABLE - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE @@ -54,7 +55,7 @@ jobs: allow_failures: - if: env(PG_BRANCH) = master - if: env(PG_BRANCH) = REL9_5_STABLE -# - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) +# - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) # Only run CI for master branch commits to limit our travis usage #branches: diff --git a/README.md b/README.md index d1ccd986..bae1171c 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `PTRACK` backup support provided via following options: * vanilla PostgreSQL 11, 12, 13, 14, 15 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13, 14, 15 -* Postgres Pro Enterprise 11, 12, 13, 14, 15 +* Postgres Pro Standard 11, 12, 13, 14 +* Postgres Pro Enterprise 11, 12, 13, 14 ## Limitations @@ -74,62 +74,62 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{15,14,13,12,11,10,9.6} +sudo apt-get source pg-probackup-{14,13,12,11,10,9.6} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6}{-dbg,} +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{15,14,13,12,11,10,9.6} +yumdownloader --source pg_probackup-{14,13,12,11,10,9.6} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11,10,9.6} -zypper install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{14,13,12,11,10,9.6} +zypper install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{15,14,13,12,11,10,9.6} +zypper si pg_probackup-{14,13,12,11,10,9.6} #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo ``` #### pg_probackup for PostgresPro Standard and Enterprise @@ -137,8 +137,8 @@ sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{15,14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{15,14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' @@ -148,35 +148,35 @@ sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). From 1a48b6c5959f75ceba18223bb7e597bce6035763 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 28 Oct 2022 19:40:15 +0300 Subject: [PATCH 49/55] [PBCKP-236] added remote shh agent path to log output --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index addd73dc..8562c85e 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -229,7 +229,7 @@ bool launch_agent(void) return false; } else { #endif - elog(LOG, "Start SSH client process, pid %d", child_pid); + elog(LOG, "Start SSH client process, pid %d, cmd \"%s\"", child_pid, cmd); SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); SYS_CHECK(close(errfd[1])); From 79009c652cd7231edf9609837a669f7d20ae67f2 Mon Sep 17 00:00:00 2001 From: MetalDream666 <61190185+MetalDream666@users.noreply.github.com> Date: Sat, 29 Oct 2022 01:17:13 +0300 Subject: [PATCH 50/55] Revert "[PBCKP-120] skip partitioned indexes for checkdb --amcheck" --- src/checkdb.c | 10 +++------- tests/checkdb.py | 9 --------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/checkdb.c b/src/checkdb.c index 1133a7b5..177fc3cc 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -461,9 +461,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' " - "AND cls.relpersistence != 't' " - "AND cls.relkind != 'I' " + "WHERE am.amname='btree' AND cls.relpersistence != 't' " "ORDER BY nmspc.nspname DESC", 0, NULL); } @@ -475,10 +473,8 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' " - "AND cls.relpersistence != 't' " - "AND cls.relkind != 'I' " - "AND (cls.reltablespace IN " + "WHERE am.amname='btree' AND cls.relpersistence != 't' AND " + "(cls.reltablespace IN " "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " "OR cls.reltablespace = 0) " "ORDER BY nmspc.nspname DESC", diff --git a/tests/checkdb.py b/tests/checkdb.py index 07b55c6d..bcda0fb2 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -38,15 +38,6 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", "create index on t_heap(id)") - - node.safe_psql( - "postgres", - "create table idxpart (a int) " - "partition by range (a)") - - node.safe_psql( - "postgres", - "create index on idxpart(a)") try: node.safe_psql( From 0b8cf419c344499d0540636a19ab944dcc167527 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Mon, 20 Jun 2022 13:44:42 +0300 Subject: [PATCH 51/55] [PBCKP-120] skip partitioned indexes for checkdb --amcheck Tags: pg_probackup --- src/checkdb.c | 10 +++++++--- tests/checkdb.py | 9 +++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/checkdb.c b/src/checkdb.c index 177fc3cc..1133a7b5 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -461,7 +461,9 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " "ORDER BY nmspc.nspname DESC", 0, NULL); } @@ -473,8 +475,10 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' AND " - "(cls.reltablespace IN " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " + "AND (cls.reltablespace IN " "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " "OR cls.reltablespace = 0) " "ORDER BY nmspc.nspname DESC", diff --git a/tests/checkdb.py b/tests/checkdb.py index bcda0fb2..07b55c6d 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -38,6 +38,15 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", "create index on t_heap(id)") + + node.safe_psql( + "postgres", + "create table idxpart (a int) " + "partition by range (a)") + + node.safe_psql( + "postgres", + "create index on idxpart(a)") try: node.safe_psql( From 0b474d261686f4554d5b853803444b3379b650ce Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 1 Nov 2022 11:53:17 +0300 Subject: [PATCH 52/55] [PBCKP-236] hotfix for C89 --- src/utils/remote.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 8562c85e..9068c940 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -321,7 +321,8 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b size_t result_size = 0; *compatibility_buf = '\0'; - for (int i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) + int i; + for (i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) { if (compatibility_params[i].strval != NULL) result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, From 64b84d0ca64a95db6154185aafc1291a6aa142df Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 1 Nov 2022 14:41:02 +0300 Subject: [PATCH 53/55] [PBCKP-236] hotfix-2 for C89 compatibility --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 9068c940..7ef8d323 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -319,9 +319,9 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b }; size_t result_size = 0; + int i; *compatibility_buf = '\0'; - int i; for (i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) { if (compatibility_params[i].strval != NULL) From feacabd8ab15129743fdd9ef287dfccd793fdfe3 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Wed, 2 Nov 2022 14:25:12 +0300 Subject: [PATCH 54/55] [PBCKP-308] Changed check_server_version function for postgresql version for 1c. --- src/backup.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 31289978..c73ee56c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -946,10 +946,21 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) */ #ifdef PGPRO_VERSION if (!res) + { /* It seems we connected to PostgreSQL (not Postgres Pro) */ - elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection is made with PostgreSQL %s", - PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + if(strcmp(PGPRO_EDITION, "1C") != 0) + { + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with PostgreSQL %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + } + /* We have PostgresPro for 1C and connect to PostgreSQL or PostgresPro for 1C + * Check the major version + */ + if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0) + elog(ERROR, "%s was built with PostgrePro %s %s, but connection is made with %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + } else { if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 && From d25a594df523163793b726ae9b096c2dbe3cface Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 3 Nov 2022 02:37:58 +0300 Subject: [PATCH 55/55] [PBCKP-235] fix one test for <15.0 --- tests/backup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 4f447c9b..6028a3ff 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -3441,10 +3441,15 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.assertIn( 'WARNING: backup in progress, stop backup', log_content) - - self.assertIn( - 'FROM pg_catalog.pg_backup_stop', - log_content) + + if self.get_version(node) < 150000: + self.assertIn( + 'FROM pg_catalog.pg_stop_backup', + log_content) + else: + self.assertIn( + 'FROM pg_catalog.pg_backup_stop', + log_content) self.assertIn( 'setting its status to ERROR',