From b38692f46490b3d47414b35b2f8d2f33f6eebcb3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jul 2019 18:01:28 +0300 Subject: [PATCH] tests: now ptrack tests by default are disabled, they can be enables via PG_PROBACKUP_PTRACK env variable. Multiple fixes, especially for PG 9.5 --- tests/Readme.md | 3 + tests/__init__.py | 5 +- tests/archive.py | 23 ++--- tests/backup.py | 174 +++++++++++++++++++++----------- tests/checkdb.py | 2 +- tests/compatibility.py | 17 ++-- tests/compression.py | 98 +++++++----------- tests/delete.py | 3 + tests/delta.py | 6 ++ tests/exclude.py | 12 +-- tests/false_positive.py | 11 +- tests/helpers/ptrack_helpers.py | 8 +- tests/logging.py | 7 +- tests/merge.py | 30 ++++-- tests/restore.py | 71 ++++++++----- tests/retention.py | 4 + 16 files changed, 284 insertions(+), 190 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index c0d9c024..d24d095c 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -29,6 +29,9 @@ Remote backup depends on key authentithication to local machine via ssh as curre Run suit of basic simple tests: export PG_PROBACKUP_TEST_BASIC=ON +Run ptrack tests: + export PG_PROBACKUP_PTRACK=ON + Usage: pip install testgres diff --git a/tests/__init__.py b/tests/__init__.py index e86bfabf..7cc96276 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -37,7 +37,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(merge)) suite.addTests(loader.loadTestsFromModule(option)) suite.addTests(loader.loadTestsFromModule(page)) -# suite.addTests(loader.loadTestsFromModule(ptrack)) suite.addTests(loader.loadTestsFromModule(pgpro560)) suite.addTests(loader.loadTestsFromModule(pgpro589)) suite.addTests(loader.loadTestsFromModule(pgpro2068)) @@ -50,6 +49,10 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(time_stamp)) suite.addTests(loader.loadTestsFromModule(validate)) + if 'PG_PROBACKUP_PTRACK' in os.environ: + if os.environ['PG_PROBACKUP_PTRACK'] == 'ON': + suite.addTests(loader.loadTestsFromModule(ptrack)) + return suite # test_pgpro434_2 unexpected success diff --git a/tests/archive.py b/tests/archive.py index 5f4aa0f8..1ce60221 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -228,10 +228,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'} - ) + initdb_params=['--data-checksums']) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -258,18 +256,19 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): log_file = os.path.join(backup_dir, 'log/pg_probackup.log') with open(log_file, 'r') as f: log_content = f.read() - self.assertNotIn( - "ERROR: pg_stop_backup doesn't answer", - log_content, - "pg_stop_backup timeouted") + + self.assertIn( + "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: log_content = f.read() - self.assertNotIn( - 'FailedAssertion', - log_content, - 'PostgreSQL crashed because of a failed assert') + + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/backup.py b/tests/backup.py index 47cf818b..d18e2d97 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -15,6 +15,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): # PGPRO-707 def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -109,6 +112,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_incremental_backup_without_full(self): """page-level backup without validated full backup""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -168,8 +174,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -230,6 +235,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_ptrack_threads(self): """ptrack multi thread backup mode""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -258,6 +266,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_ptrack_threads_stream(self): """ptrack multi thread backup mode and stream""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -287,6 +298,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_page_corruption_heal_via_ptrack_1(self): """make node, corrupt some page, check that backup failed""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -343,6 +357,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_page_corruption_heal_via_ptrack_2(self): """make node, corrupt some page, check that backup failed""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -988,6 +1005,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_drop_rel_during_backup_ptrack(self): """""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1492,10 +1512,12 @@ class BackupTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - # pg_options={'ptrack_enable': 'on'}, initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s'}) - ) + if self.ptrack: + node.append_conf('postgresql.auto.conf', 'ptrack_enable = on') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1506,58 +1528,87 @@ class BackupTest(ProbackupTest, unittest.TestCase): 'postgres', 'CREATE DATABASE backupdb') - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - # for partial restore, checkdb and ptrack - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - # for exclusive backup for PG 9.5 and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) - # ptrack functions - # for fname in [ - # 'oideq(oid, oid)', - # 'ptrack_version()', - # 'pg_ptrack_clear()', - # 'pg_ptrack_control_lsn()', - # 'pg_ptrack_get_and_clear_db(oid, oid)', - # 'pg_ptrack_get_and_clear(oid, oid)', - # 'pg_ptrack_get_block_2(oid, oid, oid, bigint)']: - # try: - # node.safe_psql( - # "backupdb", - # "GRANT EXECUTE ON FUNCTION pg_catalog.{0} " - # "TO backup".format(fname)) - # except: - # pass + if self.ptrack: + for fname in [ + 'oideq(oid, oid)', + 'ptrack_version()', + 'pg_ptrack_clear()', + 'pg_ptrack_control_lsn()', + 'pg_ptrack_get_and_clear_db(oid, oid)', + 'pg_ptrack_get_and_clear(oid, oid)', + 'pg_ptrack_get_block_2(oid, oid, oid, bigint)']: + # try: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.{0} " + "TO backup".format(fname)) + # except: + # pass # FULL backup self.backup_node( @@ -1584,12 +1635,13 @@ class BackupTest(ProbackupTest, unittest.TestCase): datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK - # self.backup_node( - # backup_dir, 'node', node, backup_type='ptrack', - # datname='backupdb', options=['-U', 'backup']) - # self.backup_node( - # backup_dir, 'node', node, backup_type='ptrack', - # datname='backupdb', options=['--stream', '-U', 'backup']) + if self.ptrack: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['--stream', '-U', 'backup']) # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/checkdb.py b/tests/checkdb.py index 7c0bdda8..1d546591 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -229,7 +229,7 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): # create two databases node.safe_psql("postgres", "create database db1") try: - node.safe_psql( + node.safe_psql( "db1", "create extension amcheck") except QueryException as e: diff --git a/tests/compatibility.py b/tests/compatibility.py index 527b4d95..7c703882 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -217,6 +217,10 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_backward_compatibility_ptrack(self): """Description in jira issue PGPRO-434""" + + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -224,8 +228,9 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off'} - ) + 'autovacuum': 'off', + 'ptrack_enable': 'on'}) + self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -262,7 +267,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Delta BACKUP with old binary + # ptrack BACKUP with old binary pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -272,7 +277,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='delta', + backup_dir, 'node', node, backup_type='ptrack', old_binary=True) if self.paranoia: @@ -287,7 +292,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Delta BACKUP with new binary + # Ptrack BACKUP with new binary pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -297,7 +302,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_dir, 'node', node, backup_type='ptrack') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/compression.py b/tests/compression.py index ef2ecf5e..ea288a5e 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -23,11 +23,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -59,15 +55,15 @@ class CompressionTest(ProbackupTest, unittest.TestCase): options=[ '--stream', '--compress-algorithm=zlib']) - # PTRACK BACKUP + # DELTA BACKUP node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=zlib']) # Drop Node @@ -105,11 +101,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.assertEqual(page_result, page_result_new) node.cleanup() - # Check ptrack backup + # Check delta backup self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + "INFO: Restore of backup {0} completed.".format(delta_backup_id), self.restore_node( - backup_dir, 'node', node, backup_id=ptrack_backup_id, + backup_dir, 'node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", "--recovery-target-action=promote"]), @@ -117,8 +113,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) node.slow_start() - ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) node.cleanup() # Clean after yourself @@ -135,11 +131,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -167,14 +159,14 @@ class CompressionTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type='page', options=["--compress-algorithm=zlib"]) - # PTRACK BACKUP + # DELTA BACKUP node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") - ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--compress-algorithm=zlib']) # Drop Node @@ -212,11 +204,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.assertEqual(page_result, page_result_new) node.cleanup() - # Check ptrack backup + # Check delta backup self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + "INFO: Restore of backup {0} completed.".format(delta_backup_id), self.restore_node( - backup_dir, 'node', node, backup_id=ptrack_backup_id, + backup_dir, 'node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", "--recovery-target-action=promote"]), @@ -224,8 +216,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) node.slow_start() - ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) node.cleanup() # Clean after yourself @@ -242,11 +234,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -275,7 +263,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type='page', options=['--stream', '--compress-algorithm=pglz']) - # PTRACK BACKUP + # DELTA BACKUP node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " @@ -283,7 +271,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "from generate_series(512,768) i") ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=pglz']) # Drop Node @@ -321,11 +309,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.assertEqual(page_result, page_result_new) node.cleanup() - # Check ptrack backup + # Check delta backup self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + "INFO: Restore of backup {0} completed.".format(delta_backup_id), self.restore_node( - backup_dir, 'node', node, backup_id=ptrack_backup_id, + backup_dir, 'node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", "--recovery-target-action=promote"]), @@ -333,8 +321,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) node.slow_start() - ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) node.cleanup() # Clean after yourself @@ -351,11 +339,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -384,15 +368,15 @@ class CompressionTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type='page', options=['--compress-algorithm=pglz']) - # PTRACK BACKUP + # DELTA BACKUP node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(200,300) i") - ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--compress-algorithm=pglz']) # Drop Node @@ -430,11 +414,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase): self.assertEqual(page_result, page_result_new) node.cleanup() - # Check ptrack backup + # Check delta backup self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + "INFO: Restore of backup {0} completed.".format(delta_backup_id), self.restore_node( - backup_dir, 'node', node, backup_id=ptrack_backup_id, + backup_dir, 'node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", "--recovery-target-action=promote"]), @@ -442,8 +426,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) node.slow_start() - ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) node.cleanup() # Clean after yourself @@ -460,11 +444,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/delete.py b/tests/delete.py index 858d8b2c..ce6bbb98 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -188,6 +188,9 @@ class DeleteTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_delete_increment_ptrack(self): """delete increment and all after him""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/tests/delta.py b/tests/delta.py index a08d62d1..6a5a9d73 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1078,6 +1078,9 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_delta_corruption_heal_via_ptrack_1(self): """make node, corrupt some page, check that backup failed""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1135,6 +1138,9 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_page_corruption_heal_via_ptrack_2(self): """make node, corrupt some page, check that backup failed""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/tests/exclude.py b/tests/exclude.py index 5e641bad..2e644d7e 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -20,10 +20,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2', - 'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -102,7 +99,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): def test_exclude_unlogged_tables_1(self): """ make node without archiving, create unlogged table, take full backup, - alter table to unlogged, take ptrack backup, restore ptrack backup, + alter table to unlogged, take delta backup, restore delta backup, check that PGDATA`s are physically the same """ fname = self.id().split('.')[3] @@ -113,8 +110,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): initdb_params=['--data-checksums'], pg_options={ 'autovacuum': 'off', - "shared_buffers": "10MB", - 'ptrack_enable': 'on'}) + "shared_buffers": "10MB"}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -138,7 +134,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): node.safe_psql('postgres', "alter table test set logged") self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + backup_dir, 'node', node, backup_type='delta', options=['--stream'] ) diff --git a/tests/false_positive.py b/tests/false_positive.py index d33309f4..d3b27f87 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -57,8 +57,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -114,6 +113,10 @@ class FalsePositive(ProbackupTest, unittest.TestCase): def test_ptrack_concurrent_get_and_clear_1(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" + + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -195,6 +198,10 @@ class FalsePositive(ProbackupTest, unittest.TestCase): def test_ptrack_concurrent_get_and_clear_2(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" + + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 7bd31a89..52edc894 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -271,7 +271,13 @@ class ProbackupTest(object): self.remote_user = None if 'PGPROBACKUP_SSH_REMOTE' in self.test_env: - self.remote = True + if self.test_env['PGPROBACKUP_SSH_REMOTE'] == 'ON': + self.remote = True + + self.ptrack = False + if 'PG_PROBACKUP_PTRACK' in self.test_env: + if self.test_env['PG_PROBACKUP_PTRACK'] == 'ON': + self.ptrack = True @property def pg_config_version(self): diff --git a/tests/logging.py b/tests/logging.py index c905695b..efde1d0b 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -142,7 +142,7 @@ class LogTest(ProbackupTest, unittest.TestCase): log_file_size) self.assertNotIn( - 'WARNING:', + 'WARNING: cannot read creation timestamp from rotation file', output) self.assertTrue(os.path.isfile(rotation_file_path)) @@ -166,7 +166,6 @@ class LogTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', options=['--log-rotation-age=1d']) - self.backup_node( backup_dir, 'node', node, options=[ @@ -212,7 +211,7 @@ class LogTest(ProbackupTest, unittest.TestCase): return_id=False) self.assertNotIn( - 'WARNING:', + 'WARNING: missing rotation file:', output) # check that log file wasn`t rotated @@ -291,7 +290,7 @@ class LogTest(ProbackupTest, unittest.TestCase): return_id=False) self.assertNotIn( - 'WARNING:', + 'WARNING: rotation file', output) # check that log file wasn`t rotated diff --git a/tests/merge.py b/tests/merge.py index 8c328ff7..e539f281 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -826,6 +826,9 @@ class MergeTest(ProbackupTest, unittest.TestCase): take page backup, merge full and page, restore last page backup and check data correctness """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1651,8 +1654,6 @@ class MergeTest(ProbackupTest, unittest.TestCase): os.remove(file_to_remove) # Try to continue failed MERGE - #print(backup_id) - #exit(1) self.merge_backup(backup_dir, "node", backup_id) self.assertEqual( @@ -1875,15 +1876,28 @@ class MergeTest(ProbackupTest, unittest.TestCase): pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum']) pgbench.wait() - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) - node.cleanup() + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored, backup_id=backup_id) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # check that merged backup has the same state as + node_restored.cleanup() self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - self.restore_node(backup_dir, 'node', node, backup_id=backup_id) - - pgdata_restored = self.pgdata_content(node.data_dir) + self.restore_node( + backup_dir, 'node', + node_restored, backup_id=backup_id) + pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself diff --git a/tests/restore.py b/tests/restore.py index 5a86b0bb..0175d19d 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -492,6 +492,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_restore_full_ptrack_archive(self): """recovery to latest from archive full+ptrack backups""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -540,6 +543,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_restore_ptrack(self): """recovery to latest from archive full+ptrack+ptrack backups""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -595,6 +601,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -647,6 +656,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): recovery to latest from full + ptrack backups with loads when ptrack backup do """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -711,6 +723,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): recovery to latest from full + page backups with loads when full backup do """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -2028,33 +2043,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.slow_start() - # Restore with recovery target lsn - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-lsn={0}'.format(target_lsn), - "--recovery-target-action=promote", - '--recovery-target-timeline=1', - ]) - - with open(recovery_conf, 'r') as f: - recovery_conf_content = f.read() - - self.assertIn( - "recovery_target_lsn = '{0}'".format(target_lsn), - recovery_conf_content) - - self.assertIn( - "recovery_target_action = 'promote'", - recovery_conf_content) - - self.assertIn( - "recovery_target_timeline = '1'", - recovery_conf_content) - - node.slow_start() - # Restore with recovery target name node.cleanup() self.restore_node( @@ -2082,6 +2070,35 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.slow_start() + # Restore with recovery target lsn + if self.get_version(node) >= 100000: + + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-lsn={0}'.format(target_lsn), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + self.assertIn( + "recovery_target_lsn = '{0}'".format(target_lsn), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/retention.py b/tests/retention.py index 1407abd9..9eb5b020 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1281,6 +1281,10 @@ class RetentionTest(ProbackupTest, unittest.TestCase): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums']) + if self.get_version(node) < 90600: + self.del_test_dir(module_name, fname) + return unittest.skip('Skipped because ptrack support is disabled') + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node)