tests: ptrack tests major revamp

tests/helpers/ptrack_helpers.py

@@ -511,10 +511,10 @@ class ProbackupTest(object):
                                 idx_dict['ptrack'][PageNum]
                             )
                         )
-
-            self.assertTrue(
-                success, 'Ptrack has failed to register changes in data files'
-            )
+            return success
+            # self.assertTrue(
+            #    success, 'Ptrack has failed to register changes in data files'
+            # )
 
     def check_ptrack_recovery(self, idx_dict):
         size = idx_dict['size']

tests/ptrack.py

@@ -1495,7 +1495,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
             node,
             idx_ptrack['path']
         )
-        self.check_ptrack_sanity(idx_ptrack)
+
+        if not self.check_ptrack_sanity(idx_ptrack):
+            self.assertTrue(
+                False, 'Ptrack has failed to register changes in data files'
+            )
 
         # GET LOGICAL CONTENT FROM NODE
         result = node.safe_psql("postgres", "select * from pgbench_accounts")

tests/ptrack_clean.py

@@ -37,7 +37,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             "as select i as id, nextval('t_seq') as t_seq, "
             "md5(i::text) as text, "
             "md5(repeat(i::text,10))::tsvector as tsvector "
-            "from generate_series(0,256) i")
+            "from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
                 node.safe_psql(
@@ -154,7 +154,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             "create sequence t_seq; create table t_heap as select i as id, "
             "nextval('t_seq') as t_seq, md5(i::text) as text, "
             "md5(repeat(i::text,10))::tsvector as tsvector "
-            "from generate_series(0,256) i")
+            "from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
                 master.safe_psql(

tests/ptrack_cluster.py

@@ -14,10 +14,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_cluster_on_btree(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -28,11 +33,18 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         node.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, nextval('t_seq') as t_seq, "
+            "md5(i::text) as text, md5(repeat(i::text,10))::tsvector "
+            "as tsvector from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
@@ -46,12 +58,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
-        self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10', '--stream'])
 
         node.safe_psql('postgres', 'delete from t_heap where id%2 = 1')
         node.safe_psql('postgres', 'cluster t_heap using t_btree')
         node.safe_psql('postgres', 'checkpoint')
 
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(node, i)
@@ -62,10 +77,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                node, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -73,10 +94,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.skip("skip")
     def test_ptrack_cluster_on_gist(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -85,11 +111,17 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         node.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap as select i as id, "
+            "nextval('t_seq') as t_seq, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
@@ -103,12 +135,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
-        self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10', '--stream'])
 
         node.safe_psql('postgres', 'delete from t_heap where id%2 = 1')
         node.safe_psql('postgres', 'cluster t_heap using t_gist')
         node.safe_psql('postgres', 'checkpoint')
 
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(node, i)
@@ -119,10 +154,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                node, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # Compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -130,10 +171,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.skip("skip")
     def test_ptrack_cluster_on_btree_replica(self):
         fname = self.id().split('.')[3]
-        master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
+        master = self.make_simple_node(
+            base_dir="{0}/{1}/master".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
@@ -141,7 +187,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
-        replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
         replica.cleanup()
 
         self.restore_node(backup_dir, 'master', replica)
@@ -154,11 +201,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         master.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap as select i as id, "
+            "nextval('t_seq') as t_seq, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
+
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                master.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'],
+                        idx_ptrack[i]['column']))
 
         master.safe_psql('postgres', 'vacuum t_heap')
         master.safe_psql('postgres', 'checkpoint')
@@ -172,13 +227,18 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
-        self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream',
-            '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])
+        self.backup_node(
+            backup_dir, 'replica', replica, options=[
+                '-j10', '--stream', '--master-host=localhost',
+                '--master-db=postgres', '--master-port={0}'.format(
+                    master.port)])
 
         master.safe_psql('postgres', 'delete from t_heap where id%2 = 1')
         master.safe_psql('postgres', 'cluster t_heap using t_btree')
         master.safe_psql('postgres', 'checkpoint')
 
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
@@ -189,10 +249,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                replica, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -200,10 +266,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.skip("skip")
     def test_ptrack_cluster_on_gist_replica(self):
         fname = self.id().split('.')[3]
-        master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
+        master = self.make_simple_node(
+            base_dir="{0}/{1}/master".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
@@ -211,7 +282,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
-        replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
         replica.cleanup()
 
         self.restore_node(backup_dir, 'master', replica)
@@ -224,15 +296,32 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         master.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap as select i as id, "
+            "nextval('t_seq') as t_seq, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
+
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                master.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'],
+                        idx_ptrack[i]['column']))
 
         master.safe_psql('postgres', 'vacuum t_heap')
         master.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
         for i in idx_ptrack:
             # get size of heap and indexes. size calculated in pages
             idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -242,13 +331,27 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
-        self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream',
-            '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])
+        self.backup_node(
+            backup_dir, 'replica', replica, options=[
+                '-j10', '--stream', '--master-host=localhost',
+                '--master-db=postgres', '--master-port={0}'.format(
+                    master.port)])
 
         master.safe_psql('postgres', 'delete from t_heap where id%2 = 1')
         master.safe_psql('postgres', 'cluster t_heap using t_gist')
         master.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
@@ -259,10 +362,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                replica, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # Compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)

tests/ptrack_move_to_tablespace.py

@@ -12,10 +12,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_recovery(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -24,19 +29,30 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         self.create_tblspace_in_node(node, 'somedata')
 
         # Create table and indexes
-        node.safe_psql("postgres",
-            "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+        node.safe_psql(
+            "postgres",
+            "create sequence t_seq; create table t_heap as select i as id, "
+            "md5(i::text) as text,md5(repeat(i::text,10))::tsvector as "
+            "tsvector from generate_series(0,2560) i")
+
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         # Move table and indexes and make checkpoint
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] == 'heap':
-                node.safe_psql('postgres', 'alter table {0} set tablespace somedata;'.format(i))
+                node.safe_psql(
+                    'postgres',
+                    'alter table {0} set tablespace somedata;'.format(i))
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql('postgres', 'alter index {0} set tablespace somedata'.format(i))
+                node.safe_psql(
+                    'postgres',
+                    'alter index {0} set tablespace somedata'.format(i))
         node.safe_psql('postgres', 'checkpoint')
 
         # Check ptrack files

tests/ptrack_recovery.py

@@ -13,10 +13,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_recovery(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -25,13 +30,21 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         self.create_tblspace_in_node(node, 'somedata')
 
         # Create table
-        node.safe_psql("postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+        node.safe_psql(
+            "postgres",
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
+
         # Create indexes
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres", "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
             # get size of heap and indexes. size calculated in pages
             idx_ptrack[i]['size'] = int(self.get_fork_size(node, i))

tests/ptrack_truncate.py

@@ -12,10 +12,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_truncate(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -26,11 +31,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         node.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
+
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         node.safe_psql('postgres', 'truncate t_heap')
         node.safe_psql('postgres', 'checkpoint')
@@ -45,7 +58,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
         # Make full backup to clean every ptrack
-        self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10', '--stream'])
         for i in idx_ptrack:
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
                 node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
@@ -57,10 +71,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.skip("skip")
     def test_ptrack_truncate_replica(self):
         fname = self.id().split('.')[3]
-        master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
+        master = self.make_simple_node(
+            base_dir="{0}/{1}/master".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2',
+                'checkpoint_timeout': '30'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
@@ -68,7 +88,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
-        replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
         replica.cleanup()
 
         self.restore_node(backup_dir, 'master', replica)
@@ -81,15 +102,30 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         master.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                master.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                master.safe_psql(
+                    "postgres", "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         replica.safe_psql('postgres', 'truncate t_heap')
         replica.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
         for i in idx_ptrack:
             # get fork size and calculate it in pages
             idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -100,7 +136,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
         # Make full backup to clean every ptrack
-        self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'replica', replica, options=['-j10', '--stream'])
         for i in idx_ptrack:
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
                 replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
@@ -111,6 +148,17 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'vacuum t_heap')
         master.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes and calculate it in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
@@ -121,10 +169,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                replica, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)

tests/ptrack_vacuum.py

@@ -12,10 +12,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_vacuum(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -26,11 +31,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         node.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'],
+                        idx_ptrack[i]['column']))
 
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
@@ -45,7 +58,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
         # Make full backup to clean every ptrack
-        self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10', '--stream'])
         for i in idx_ptrack:
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
                 node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
@@ -56,6 +70,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes and calculate it in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(node, i)
@@ -66,10 +82,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                node, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -77,10 +99,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.skip("skip")
     def test_ptrack_vacuum_replica(self):
         fname = self.id().split('.')[3]
-        master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
+        master = self.make_simple_node(
+            base_dir="{0}/{1}/master".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2',
+                'checkpoint_timeout': '30'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
@@ -88,7 +116,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
-        replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
         replica.cleanup()
 
         self.restore_node(backup_dir, 'master', replica)
@@ -101,15 +130,30 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         master.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap as select i as id, "
+            "md5(i::text) as text, md5(repeat(i::text,10))::tsvector "
+            "as tsvector from generate_series(0,2560) i")
+
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                master.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         master.safe_psql('postgres', 'vacuum t_heap')
         master.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
         for i in idx_ptrack:
             # get fork size and calculate it in pages
             idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -120,8 +164,12 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
         # Make FULL backup to clean every ptrack
-        self.backup_node(backup_dir, 'replica', replica, options=['-j10',
-            '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])
+        self.backup_node(
+            backup_dir, 'replica', replica, options=[
+                '-j10', '--master-host=localhost',
+                '--master-db=postgres',
+                '--master-port={0}'.format(master.port)])
+
         for i in idx_ptrack:
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
                 replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
@@ -132,7 +180,17 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'vacuum t_heap')
         master.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
         # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes and calculate it in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
@@ -143,10 +201,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                replica, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)

tests/ptrack_vacuum_bits_frozen.py

@@ -12,10 +12,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_vacuum_bits_frozen(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -26,11 +31,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         res = node.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'],
+                        idx_ptrack[i]['column']))
 
         node.safe_psql('postgres', 'checkpoint')
 
@@ -43,11 +56,14 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
-        self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10', '--stream'])
 
         node.safe_psql('postgres', 'vacuum freeze t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(node, i)
@@ -58,10 +74,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                node, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -69,10 +91,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.skip("skip")
     def test_ptrack_vacuum_bits_frozen_replica(self):
         fname = self.id().split('.')[3]
-        master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
+        master = self.make_simple_node(
+            base_dir="{0}/{1}/master".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
@@ -80,7 +107,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
-        replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
         replica.cleanup()
 
         self.restore_node(backup_dir, 'master', replica)
@@ -93,14 +121,29 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         master.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap as select i as id, "
+            "md5(i::text) as text, md5(repeat(i::text,10))::tsvector "
+            "as tsvector from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                master.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'],
+                        idx_ptrack[i]['column']))
 
         master.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
         for i in idx_ptrack:
             # get size of heap and indexes. size calculated in pages
             idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -111,12 +154,28 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
         # Take PTRACK backup to clean every ptrack
-        self.backup_node(backup_dir, 'replica', replica, options=['-j10',
-            '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])
+        self.backup_node(
+            backup_dir, 'replica', replica,
+            options=[
+                '-j10',
+                '--master-host=localhost',
+                '--master-db=postgres',
+                '--master-port={0}'.format(master.port)])
 
         master.safe_psql('postgres', 'vacuum freeze t_heap')
         master.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
@@ -127,10 +186,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                replica, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)

tests/ptrack_vacuum_bits_visibility.py

@@ -12,10 +12,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_vacuum_bits_visibility(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -26,14 +31,31 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         res = node.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
+
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         node.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
         for i in idx_ptrack:
             # get size of heap and indexes. size calculated in pages
             idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@@ -43,11 +65,23 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
-        self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10', '--stream'])
 
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(node, i)
@@ -58,10 +92,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                node, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)

tests/ptrack_vacuum_full.py

@@ -35,7 +35,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             "create sequence t_seq; create table t_heap tablespace somedata "
             "as select i as id, md5(i::text) as text, "
             "md5(repeat(i::text,10))::tsvector as tsvector "
-            "from generate_series(0,127) i")
+            "from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
                 node.safe_psql(
@@ -63,6 +63,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'vacuum full t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(node, i)
@@ -77,7 +79,12 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity, the most important part
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -117,7 +124,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             "postgres",
             "create sequence t_seq; create table t_heap as select i as id, "
             "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as "
-            "tsvector from generate_series(0,127) i")
+            "tsvector from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
                 master.safe_psql(
@@ -133,8 +140,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Sync master and replica
         lsn = master.safe_psql(
             'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-#        print(lsn)
-
         replica.poll_query_until(
             "postgres",
             "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
@@ -168,18 +173,14 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Sync master and replica
         lsn = master.safe_psql(
             'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
-#        print(lsn)
-
         replica.poll_query_until(
             "postgres",
             "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
                 lsn))
-
         replica.safe_psql('postgres', 'checkpoint')
 
-#        print(replica.safe_psql(
-#            'postgres', 'SELECT pg_catalog.pg_last_wal_replay_lsn()'))
-
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
@@ -194,7 +195,12 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity, the most important part
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+                success, 'Ptrack has failed to register changes in data files'
+            )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)

tests/ptrack_vacuum_truncate.py

@@ -12,10 +12,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_vacuum_truncate(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -26,11 +31,17 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         res = node.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,2560) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres", "create index {0} on {1} using {2}({3}) "
+                    "tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
@@ -44,12 +55,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
-        self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10', '--stream'])
 
         node.safe_psql('postgres', 'delete from t_heap where id > 128;')
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(node, i)
@@ -60,23 +74,34 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                node, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-
     # @unittest.skip("skip")
     # @unittest.expectedFailure
     def test_ptrack_vacuum_truncate_replica(self):
         fname = self.id().split('.')[3]
-        master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
+        master = self.make_simple_node(
+            base_dir="{0}/{1}/master".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2',
+                'checkpoint_timeout': '30'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
@@ -84,7 +109,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
 
-        replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
         replica.cleanup()
 
         self.restore_node(backup_dir, 'master', replica)
@@ -97,11 +123,17 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         master.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+            "create sequence t_seq; create table t_heap as select i as id, "
+            "md5(i::text) as text, md5(repeat(i::text,10))::tsvector "
+            "as tsvector from generate_series(0,2560) i")
+
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                master.safe_psql(
+                    "postgres", "create index {0} on {1} "
+                    "using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         master.safe_psql('postgres', 'vacuum t_heap')
         master.safe_psql('postgres', 'checkpoint')
@@ -130,6 +162,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'vacuum t_heap')
         master.safe_psql('postgres', 'checkpoint')
 
+        # CHECK PTRACK SANITY
+        success = True
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
             idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
@@ -140,10 +174,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                replica, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity
-            self.check_ptrack_sanity(idx_ptrack[i])
+            if not self.check_ptrack_sanity(idx_ptrack[i]):
+                success = False
+
+        self.assertTrue(
+            success, 'Ptrack has failed to register changes in data files'
+        )
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)