tests: add module ptrack_empty, create_db_on_replica

tests/__init__.py

@@ -2,7 +2,7 @@ import unittest
 
 from . import init_test, merge, option_test, show_test, compatibility, \
     backup_test, delete_test, delta, restore_test, validate_test, \
-    retention_test, ptrack_clean, ptrack_cluster, \
+    retention_test, ptrack_clean, ptrack_empty, ptrack_cluster, \
     ptrack_move_to_tablespace, ptrack_recovery, ptrack_truncate, \
     ptrack_vacuum, ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \
     ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \
@@ -31,6 +31,7 @@ def load_tests(loader, tests, pattern):
     suite.addTests(loader.loadTestsFromModule(page))
     suite.addTests(loader.loadTestsFromModule(ptrack))
     suite.addTests(loader.loadTestsFromModule(ptrack_clean))
+    suite.addTests(loader.loadTestsFromModule(ptrack_empty))
     suite.addTests(loader.loadTestsFromModule(ptrack_cluster))
     suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace))
     suite.addTests(loader.loadTestsFromModule(ptrack_recovery))

tests/helpers/ptrack_helpers.py

@@ -48,6 +48,11 @@ idx_ptrack = {
         'column': 'tsvector',
         'relation': 't_heap'
     },
+    't_hash': {
+        'type': 'hash',
+        'column': 'id',
+        'relation': 't_heap'
+    }
 }
 
 archive_script = """
@@ -1002,6 +1007,27 @@ class ProbackupTest(object):
             else:
                 node.execute("select pg_switch_xlog()")
 
+    def wait_until_replica_catch_with_master(self, master, replica):
+
+        if self.version_to_num(
+                master.safe_psql(
+                    "postgres",
+                    "show server_version")) >= self.version_to_num('10.0'):
+            master_function = 'pg_catalog.pg_current_wal_lsn()'
+            replica_function = 'pg_catalog.pg_last_wal_replay_lsn()'
+        else:
+            master_function = 'pg_catalog.pg_current_xlog_location()'
+            replica_function = 'pg_catalog.pg_last_xlog_replay_location()'
+
+        lsn = master.safe_psql(
+            'postgres',
+            'SELECT {0}'.format(master_function)).rstrip()
+
+        # Wait until replica catch up with master
+        replica.poll_query_until(
+            'postgres',
+            "SELECT '{0}'::pg_lsn <= {1}".format(lsn, replica_function))
+
     def get_version(self, node):
         return self.version_to_num(
             testgres.get_pg_config()["VERSION"].split(" ")[1])

tests/ptrack.py

@@ -32,7 +32,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # PTRACK BACKUP
         try:
@@ -82,7 +82,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         self.backup_node(backup_dir, 'node', node, options=['--stream'])
@@ -146,7 +146,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
         node_restored.cleanup()
-        node.start()
+        node.slow_start()
 
         self.backup_node(backup_dir, 'node', node)
         con = node.connect("postgres")
@@ -178,7 +178,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node_restored.append_conf(
             "postgresql.auto.conf", "port = {0}".format(node_restored.port))
 
-        node_restored.start()
+        node_restored.slow_start()
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -208,7 +208,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
         node_restored.cleanup()
-        node.start()
+        node.slow_start()
         self.create_tblspace_in_node(node, 'somedata')
 
         self.backup_node(backup_dir, 'node', node)
@@ -275,7 +275,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node_restored.append_conf(
             "postgresql.auto.conf", "port = {0}".format(node_restored.port))
 
-        node_restored.start()
+        node_restored.slow_start()
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -308,7 +308,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
         node_restored.cleanup()
-        node.start()
+        node.slow_start()
         self.create_tblspace_in_node(node, 'somedata')
 
         node.safe_psql(
@@ -367,7 +367,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         node_restored.append_conf(
             "postgresql.auto.conf", "port = {0}".format(node_restored.port))
-        node_restored.start()
+        node_restored.slow_start()
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -397,7 +397,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
         node_restored.cleanup()
-        node.start()
+        node.slow_start()
 
         self.backup_node(backup_dir, 'node', node)
 
@@ -437,7 +437,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         node_restored.append_conf(
             "postgresql.auto.conf", "port = {0}".format(node_restored.port))
-        node_restored.start()
+        node_restored.slow_start()
 
         # Logical comparison
         self.assertEqual(
@@ -468,7 +468,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         node.safe_psql(
             "postgres",
@@ -509,7 +509,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
                 node.data_dir, ignore_ptrack=False)
             self.compare_pgdata(pgdata, pgdata_restored)
 
-        node.start()
+        node.slow_start()
         # Logical comparison
         self.assertEqual(
             result,
@@ -541,7 +541,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         node.safe_psql("postgres", "create sequence t_seq")
@@ -637,7 +637,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         node.safe_psql(
@@ -743,7 +743,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         node.safe_psql(
@@ -827,7 +827,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         node.safe_psql(
@@ -898,7 +898,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         node.safe_psql(
@@ -978,7 +978,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         node.safe_psql(
@@ -1027,7 +1027,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         # START RESTORED NODE
         node_restored.append_conf(
             "postgresql.auto.conf", "port = {0}".format(node_restored.port))
-        node_restored.start()
+        node_restored.slow_start()
 
         # DROP DATABASE DB1
         node.safe_psql(
@@ -1057,7 +1057,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         # START RESTORED NODE
         node_restored.append_conf(
             "postgresql.auto.conf", "port = {0}".format(node_restored.port))
-        node_restored.start()
+        node_restored.slow_start()
 
         try:
             node_restored.safe_psql('db1', 'select 1')
@@ -1078,6 +1078,110 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
+    # @unittest.skip("skip")
+    def test_create_db_on_replica(self):
+        """
+        Make node, take full backup, create replica from it,
+        take full backup from replica,
+        create database db1, take ptrack backup from replica,
+        restore database and check it presense
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={
+                'wal_level': 'replica',
+                'max_wal_senders': '2',
+                'checkpoint_timeout': '30s',
+                'ptrack_enable': 'on',
+                'autovacuum': 'off'
+            }
+        )
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        # FULL BACKUP
+        node.safe_psql(
+            "postgres",
+            "create table t_heap as select i as id, md5(i::text) as text, "
+            "md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
+
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica.cleanup()
+
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10'])
+
+        self.restore_node(backup_dir, 'node', replica)
+
+        # Add replica
+        self.add_instance(backup_dir, 'replica', replica)
+        self.set_replica(node, replica, 'replica', synchronous=True)
+        self.set_archiving(backup_dir, 'replica', replica, replica=True)
+        replica.slow_start(replica=True)
+
+        self.backup_node(
+            backup_dir, 'replica', replica,
+            options=[
+                '-j10',
+                '--master-host=localhost',
+                '--master-db=postgres',
+                '--master-port={0}'.format(node.port)
+                ]
+            )
+
+        # CREATE DATABASE DB1
+        node.safe_psql("postgres", "create database db1")
+        node.safe_psql(
+            "db1",
+            "create table t_heap as select i as id, md5(i::text) as text, "
+            "md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
+
+        # Wait until replica catch up with master
+        self.wait_until_replica_catch_with_master(node, replica)
+        replica.safe_psql('postgres', 'checkpoint')
+
+        # PTRACK BACKUP
+        backup_id = self.backup_node(
+            backup_dir, 'replica',
+            replica, backup_type='ptrack',
+            options=[
+                '-j10',
+                '--master-host=localhost',
+                '--master-db=postgres',
+                '--master-port={0}'.format(node.port)
+                ]
+            )
+
+        if self.paranoia:
+            pgdata = self.pgdata_content(replica.data_dir)
+
+        # RESTORE
+        node_restored = self.make_simple_node(
+            base_dir="{0}/{1}/node_restored".format(module_name, fname)
+        )
+        node_restored.cleanup()
+
+        self.restore_node(
+            backup_dir, 'replica', node_restored,
+            backup_id=backup_id, options=["-j", "4"])
+
+        # COMPARE PHYSICAL CONTENT
+        if self.paranoia:
+            pgdata_restored = self.pgdata_content(
+                node_restored.data_dir)
+            self.compare_pgdata(pgdata, pgdata_restored)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
     # @unittest.skip("skip")
     def test_alter_table_set_tablespace_ptrack(self):
         """Make node, create tablespace with table, take full backup,
@@ -1098,7 +1202,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         self.create_tblspace_in_node(node, 'somedata')
@@ -1195,7 +1299,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # FULL BACKUP
         self.backup_node(backup_dir, 'node', node, options=["--stream"])
@@ -1237,7 +1341,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
             self.compare_pgdata(pgdata, pgdata_restored)
 
         # START RESTORED NODE
-        node_restored.start()
+        node_restored.slow_start()
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -1265,7 +1369,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
 
@@ -1307,7 +1411,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.cleanup()
         shutil.rmtree(tblspace, ignore_errors=True)
         self.restore_node(backup_dir, 'node', node, options=["-j", "4"])
-        node.start()
+        node.slow_start()
 
         tblspc_exist = node.safe_psql(
             "postgres",
@@ -1347,7 +1451,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
         tblspc_path = self.get_tblspace_path(node, 'somedata')
@@ -1463,7 +1567,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
 
@@ -1566,7 +1670,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         # Take FULL backup to clean every ptrack
         self.backup_node(

tests/ptrack_cluster.py

@@ -238,12 +238,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY
@@ -323,12 +318,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         self.backup_node(
@@ -351,12 +341,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY

tests/ptrack_empty.py

@@ -0,0 +1,181 @@
+import os
+import unittest
+from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
+import time
+
+
+module_name = 'ptrack_clean'
+
+
+class SimpleTest(ProbackupTest, unittest.TestCase):
+
+    # @unittest.skip("skip")
+    # @unittest.expectedFailure
+    def test_ptrack_clean(self):
+        """Take backups of every available types and check that PTRACK is clean"""
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2',
+                'autovacuum': 'off'})
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.start()
+
+        self.create_tblspace_in_node(node, 'somedata')
+
+        # Create table
+        node.safe_psql(
+            "postgres",
+            "create sequence t_seq; create table t_heap "
+            "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector) "
+            "tablespace somedata")
+
+        # Take FULL backup to clean every ptrack
+        self.backup_node(
+            backup_dir, 'node', node,
+            options=['-j10', '--stream'])
+
+        # Create indexes
+        for i in idx_ptrack:
+            if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
+                node.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'],
+                        idx_ptrack[i]['column']))
+
+        node.safe_psql('postgres', 'checkpoint')
+
+        node_restored = self.make_simple_node(
+            base_dir="{0}/{1}/node_restored".format(module_name, fname))
+        node_restored.cleanup()
+
+        tblspace1 = self.get_tblspace_path(node, 'somedata')
+        tblspace2 = self.get_tblspace_path(node_restored, 'somedata')
+
+        # Take PTRACK backup
+        backup_id = self.backup_node(
+            backup_dir, 'node', node, backup_type='ptrack',
+            options=['-j10', '--log-level-file=verbose'])
+
+        if self.paranoia:
+            pgdata = self.pgdata_content(node.data_dir)
+
+        self.restore_node(
+            backup_dir, 'node', node_restored,
+            backup_id=backup_id,
+            options=[
+                "-j", "4",
+                "-T{0}={1}".format(tblspace1, tblspace2)]
+        )
+
+        if self.paranoia:
+            pgdata_restored = self.pgdata_content(node_restored.data_dir)
+            self.compare_pgdata(pgdata, pgdata_restored)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    # @unittest.expectedFailure
+    def test_ptrack_clean_replica(self):
+        """Take backups of every available types from master and check that PTRACK on replica is clean"""
+        fname = self.id().split('.')[3]
+        master = self.make_simple_node(
+            base_dir="{0}/{1}/master".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'master', master)
+        master.start()
+
+        self.backup_node(backup_dir, 'master', master, options=['--stream'])
+
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica.cleanup()
+
+        self.restore_node(backup_dir, 'master', replica)
+
+        self.add_instance(backup_dir, 'replica', replica)
+        self.set_replica(master, replica, synchronous=True)
+        self.set_archiving(backup_dir, 'replica', replica, replica=True)
+        replica.start()
+
+        # Create table
+        master.safe_psql(
+            "postgres",
+            "create sequence t_seq; create table t_heap "
+            "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector)")
+        self.wait_until_replica_catch_with_master(master, replica)
+
+        # Take FULL backup
+        self.backup_node(
+            backup_dir,
+            'replica',
+            replica,
+            options=[
+                '-j10', '--stream',
+                '--master-host=localhost',
+                '--master-db=postgres',
+                '--master-port={0}'.format(master.port)])
+
+        # Create indexes
+        for i in idx_ptrack:
+            if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
+                master.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'],
+                        idx_ptrack[i]['column']))
+
+
+        self.wait_until_replica_catch_with_master(master, replica)
+
+        node_restored = self.make_simple_node(
+            base_dir="{0}/{1}/node_restored".format(module_name, fname))
+        node_restored.cleanup()
+
+        # Take PTRACK backup
+        backup_id = self.backup_node(
+            backup_dir,
+            'replica',
+            replica,
+            backup_type='ptrack',
+            options=[
+                '-j10', '--stream',
+                '--master-host=localhost',
+                '--master-db=postgres',
+                '--master-port={0}'.format(master.port)])
+        
+        if self.paranoia:
+            pgdata = self.pgdata_content(replica.data_dir)
+
+        self.restore_node(
+            backup_dir, 'replica', node_restored,
+            backup_id=backup_id,
+            options=["-j", "4"]
+        )
+
+        if self.paranoia:
+            pgdata_restored = self.pgdata_content(node_restored.data_dir)
+            self.compare_pgdata(pgdata, pgdata_restored)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)

tests/ptrack_truncate.py

@@ -24,7 +24,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
 
@@ -85,7 +85,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
-        master.start()
+        master.slow_start()
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
@@ -98,7 +98,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'replica', replica)
         self.set_replica(master, replica, 'replica', synchronous=True)
         self.set_archiving(backup_dir, 'replica', replica, replica=True)
-        replica.start()
+        replica.slow_start(replica=True)
 
         # Create table and indexes
         self.create_tblspace_in_node(master, 'somedata')
@@ -117,12 +117,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                         idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         for i in idx_ptrack:
@@ -152,12 +147,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY

tests/ptrack_vacuum.py

@@ -24,7 +24,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
 
@@ -112,7 +112,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
-        master.start()
+        master.slow_start()
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
@@ -125,7 +125,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'replica', replica)
         self.set_replica(master, replica, 'replica', synchronous=True)
         self.set_archiving(backup_dir, 'replica', replica, replica=True)
-        replica.start()
+        replica.slow_start(replica=True)
 
         # Create table and indexes
         master.safe_psql(
@@ -146,12 +146,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # Make FULL backup to clean every ptrack
@@ -181,12 +176,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY

tests/ptrack_vacuum_bits_frozen.py

@@ -24,7 +24,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
 
@@ -103,7 +103,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
-        master.start()
+        master.slow_start()
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
@@ -116,7 +116,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'replica', replica)
         self.set_replica(master, replica, synchronous=True)
         self.set_archiving(backup_dir, 'replica', replica, replica=True)
-        replica.start()
+        replica.slow_start(replica=True)
 
         # Create table and indexes
         master.safe_psql(
@@ -136,12 +136,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # Take PTRACK backup to clean every ptrack
@@ -166,12 +161,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY

tests/ptrack_vacuum_bits_visibility.py

@@ -24,7 +24,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
 

tests/ptrack_vacuum_full.py

@@ -25,7 +25,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
 
@@ -105,7 +105,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
-        master.start()
+        master.slow_start()
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
         replica = self.make_simple_node(
@@ -117,7 +117,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'replica', replica)
         self.set_replica(master, replica, 'replica', synchronous=True)
         self.set_archiving(backup_dir, 'replica', replica, replica=True)
-        replica.start()
+        replica.slow_start(replica=True)
 
         # Create table and indexes
         master.safe_psql(
@@ -138,12 +138,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # Take FULL backup to clean every ptrack
@@ -171,12 +166,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'checkpoint')
 
         # Sync master and replica
-        lsn = master.safe_psql(
-            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-        replica.poll_query_until(
-            "postgres",
-            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
-                lsn))
+        self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY

tests/ptrack_vacuum_truncate.py

@@ -24,7 +24,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
-        node.start()
+        node.slow_start()
 
         self.create_tblspace_in_node(node, 'somedata')
 
@@ -105,7 +105,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
-        master.start()
+        master.slow_start()
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
@@ -118,7 +118,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'replica', replica)
         self.set_replica(master, replica, 'replica', synchronous=True)
         self.set_archiving(backup_dir, 'replica', replica, replica=True)
-        replica.start()
+        replica.slow_start(replica=True)
 
         # Create table and indexes
         master.safe_psql(