tests: fix major bugs in ptrack tests

tests/helpers/ptrack_helpers.py

@@ -460,7 +460,7 @@ class ProbackupTest(object):
                     if self.verbose:
                         print(
                             'File: {0}\n Page Number {1} of type {2} was deleted,'
-                            ' but ptrack value is {3}'.format(
+                            ' but ptrack value is {3}. THIS IS BAD'.format(
                                 idx_dict['path'],
                                 PageNum, idx_dict['type'],
                                 idx_dict['ptrack'][PageNum])

tests/ptrack_vacuum_full.py

@@ -1,6 +1,7 @@
 import os
 import unittest
 from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
+import time
 
 
 module_name = 'ptrack_vacuum_full'
@@ -12,10 +13,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_vacuum_full(self):
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on',
+                'wal_level': 'replica',
+                'max_wal_senders': '2'})
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -26,11 +32,17 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         res = node.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i")
+            "create sequence t_seq; create table t_heap tablespace somedata "
+            "as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,127) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                node.safe_psql(
+                    "postgres", "create index {0} on {1} "
+                    "using {2}({3}) tablespace somedata".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
@@ -44,7 +56,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
             idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
-        self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
+        self.backup_node(
+            backup_dir, 'node', node, options=['-j10', '--stream'])
 
         node.safe_psql('postgres', 'delete from t_heap where id%2 = 1')
         node.safe_psql('postgres', 'vacuum full t_heap')
@@ -60,7 +73,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                node, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity, the most important part
             self.check_ptrack_sanity(idx_ptrack[i])
@@ -72,17 +86,23 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
     # @unittest.expectedFailure
     def test_ptrack_vacuum_full_replica(self):
         fname = self.id().split('.')[3]
-        master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
+        master = self.make_simple_node(
+            base_dir="{0}/{1}/master".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
+            pg_options={
+                'ptrack_enable': 'on', 'wal_level': 'replica',
+                'max_wal_senders': '2', 'autovacuum': 'off',
+                'checkpoint_timeout': '30s'}
+            )
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
         master.start()
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
-        replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica = self.make_simple_node(
+            base_dir="{0}/{1}/replica".format(module_name, fname))
         replica.cleanup()
 
         self.restore_node(backup_dir, 'master', replica)
@@ -95,15 +115,32 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
         # Create table and indexes
         master.safe_psql(
             "postgres",
-            "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i")
+            "create sequence t_seq; create table t_heap as select i as id, "
+            "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as "
+            "tsvector from generate_series(0,127) i")
         for i in idx_ptrack:
             if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
-                master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
-                    i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
+                master.safe_psql(
+                    "postgres",
+                    "create index {0} on {1} using {2}({3})".format(
+                        i, idx_ptrack[i]['relation'],
+                        idx_ptrack[i]['type'],
+                        idx_ptrack[i]['column']))
 
         master.safe_psql('postgres', 'vacuum t_heap')
         master.safe_psql('postgres', 'checkpoint')
 
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+#        print(lsn)
+
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+        replica.safe_psql('postgres', 'checkpoint')
+
         for i in idx_ptrack:
             # get size of heap and indexes. size calculated in pages
             idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -114,12 +151,34 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
         # Take FULL backup to clean every ptrack
-        self.backup_node(backup_dir, 'replica', replica, options=['-j10',
-            '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])
+        self.backup_node(
+            backup_dir, 'replica', replica,
+            options=[
+                '-j10',
+                '--master-host=localhost',
+                '--master-db=postgres',
+                '--master-port={0}'.format(master.port)
+                ]
+            )
+        # TODO: check that all ptrack are nullified
 
         master.safe_psql('postgres', 'delete from t_heap where id%2 = 1')
         master.safe_psql('postgres', 'vacuum full t_heap')
-        master.safe_psql('postgres', 'checkpoint')
+
+        # Sync master and replica
+        lsn = master.safe_psql(
+            'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
+#        print(lsn)
+
+        replica.poll_query_until(
+            "postgres",
+            "SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
+                lsn))
+
+        replica.safe_psql('postgres', 'checkpoint')
+
+#        print(replica.safe_psql(
+#            'postgres', 'SELECT pg_catalog.pg_last_wal_replay_lsn()'))
 
         for i in idx_ptrack:
             # get new size of heap and indexes. size calculated in pages
@@ -131,7 +190,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
             # get ptrack for every idx
             idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
-                replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
+                replica, idx_ptrack[i]['path'],
+                [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
 
             # compare pages and check ptrack sanity, the most important part
             self.check_ptrack_sanity(idx_ptrack[i])

tests/ptrack_vacuum_truncate.py

@@ -116,8 +116,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
                 idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
         # Take PTRACK backup to clean every ptrack
-        self.backup_node(backup_dir, 'replica', replica, options=['-j10',
-            '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])
+        self.backup_node(
+            backup_dir, 'replica', replica,
+            options=[
+                '-j10',
+                '--master-host=localhost',
+                '--master-db=postgres',
+                '--master-port={0}'.format(master.port)
+                ]
+            )
 
         master.safe_psql('postgres', 'delete from t_heap where id > 128;')
         master.safe_psql('postgres', 'vacuum t_heap')