tests: ptrack2 support for PG11

tests/ptrack.py

@@ -54,6 +54,161 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
+        # @unittest.skip("skip")
+    def test_ptrack_multi_timeline_backup(self):
+        """
+        t2            /------P2
+        t1 ------F---*-----P1
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            ptrack_enable=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'autovacuum': 'off'})
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        if node.major_version >= 11:
+            node.safe_psql(
+                "postgres",
+                "CREATE EXTENSION ptrack")
+
+        node.pgbench_init(scale=5)
+
+        # FULL backup
+        full_id = self.backup_node(backup_dir, 'node', node)
+
+        pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
+        sleep(15)
+
+        xid = node.safe_psql(
+            'postgres',
+            'SELECT txid_current()').rstrip()
+        pgbench.wait()
+
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack')
+
+        node.cleanup()
+
+        # Restore from full backup to create Timeline 2
+        print(self.restore_node(
+            backup_dir, 'node', node,
+            options=[
+                '--recovery-target-xid={0}'.format(xid),
+                '--recovery-target-action=promote']))
+
+        node.slow_start()
+
+        pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum'])
+        pgbench.wait()
+
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack')
+
+        pgdata = self.pgdata_content(node.data_dir)
+
+        node.cleanup()
+
+        self.restore_node(backup_dir, 'node', node)
+
+        pgdata_restored = self.pgdata_content(node.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
+
+        node.slow_start()
+
+        balance = node.safe_psql(
+            'postgres',
+            'select (select sum(tbalance) from pgbench_tellers) - '
+            '( select sum(bbalance) from pgbench_branches) + '
+            '( select sum(abalance) from pgbench_accounts ) - '
+            '(select sum(delta) from pgbench_history) as must_be_zero').rstrip()
+
+        self.assertEqual('0', balance)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+        # @unittest.skip("skip")
+    def test_ptrack_multi_timeline_backup_1(self):
+        """
+        t2              /------
+        t1 ---F---P1---*
+
+        # delete P1
+        t2              /------P2
+        t1 ---F--------*
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            ptrack_enable=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'autovacuum': 'off'})
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        if node.major_version >= 11:
+            node.safe_psql(
+                "postgres",
+                "CREATE EXTENSION ptrack")
+
+        node.pgbench_init(scale=5)
+
+        # FULL backup
+        full_id = self.backup_node(backup_dir, 'node', node)
+
+        pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum'])
+        pgbench.wait()
+
+        ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack')
+        node.cleanup()
+
+        self.restore_node(backup_dir, 'node', node)
+
+        node.slow_start()
+
+        pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum'])
+        pgbench.wait()
+
+        # delete old PTRACK backup
+        self.delete_pb(backup_dir, 'node', backup_id=ptrack_id)
+
+        # take new PTRACK backup
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack')
+
+        pgdata = self.pgdata_content(node.data_dir)
+
+        node.cleanup()
+
+        self.restore_node(backup_dir, 'node', node)
+
+        pgdata_restored = self.pgdata_content(node.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
+
+        node.slow_start()
+
+        balance = node.safe_psql(
+            'postgres',
+            'select (select sum(tbalance) from pgbench_tellers) - '
+            '( select sum(bbalance) from pgbench_branches) + '
+            '( select sum(abalance) from pgbench_accounts ) - '
+            '(select sum(delta) from pgbench_history) as must_be_zero').rstrip()
+
+        self.assertEqual('0', balance)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
     # @unittest.skip("skip")
     def test_ptrack_eat_my_data(self):
         """
@@ -330,7 +485,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
                 "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
             )
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             fnames = [
                 'pg_catalog.oideq(oid, oid)',
                 'pg_catalog.ptrack_version()',
@@ -1868,7 +2023,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.backup_node(backup_dir, 'node', node, options=['--stream'])
 
         # PTRACK STUFF
-        if node.major_version < 12:
+        if node.major_version < 11:
             idx_ptrack = {'type': 'heap'}
             idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts')
             idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts')
@@ -1881,7 +2036,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         node.safe_psql("postgres", "checkpoint")
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             idx_ptrack['new_size'] = self.get_fork_size(
                 node,
                 'pgbench_accounts')
@@ -2299,7 +2454,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'vacuum t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 # get size of heap and indexes. size calculated in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@@ -2317,7 +2472,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY
-        if node.major_version < 12:
+        if node.major_version < 11:
             self.check_ptrack_map_sanity(node, idx_ptrack)
 
         # Clean after yourself
@@ -2378,7 +2533,20 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY
-        self.check_ptrack_map_sanity(node, idx_ptrack)
+        if node.major_version < 11:
+            self.check_ptrack_map_sanity(node, idx_ptrack)
+
+        self.backup_node(
+            backup_dir, 'node', node,
+            backup_type='ptrack', options=['-j10', '--stream'])
+
+        pgdata = self.pgdata_content(node.data_dir)
+        node.cleanup()
+
+        self.restore_node(backup_dir, 'node', node)
+
+        pgdata_restored = self.pgdata_content(node.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -2397,7 +2565,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'master', master)
         master.slow_start()
 
-        if master.major_version >= 12:
+        if master.major_version >= 11:
             master.safe_psql(
                 "postgres",
                 "CREATE EXTENSION ptrack")
@@ -2459,7 +2627,23 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY
-        self.check_ptrack_map_sanity(replica, idx_ptrack)
+        if master.major_version < 11:
+            self.check_ptrack_map_sanity(replica, idx_ptrack)
+
+        self.backup_node(
+            backup_dir, 'replica', replica,
+            backup_type='ptrack', options=['-j10', '--stream'])
+
+        pgdata = self.pgdata_content(replica.data_dir)
+
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'))
+        node.cleanup()
+
+        self.restore_node(backup_dir, 'replica', node)
+
+        pgdata_restored = self.pgdata_content(replica.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -2477,7 +2661,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'master', master)
         master.slow_start()
 
-        if master.major_version >= 12:
+        if master.major_version >= 11:
             master.safe_psql(
                 "postgres",
                 "CREATE EXTENSION ptrack")
@@ -2537,13 +2721,13 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id%2 = 1')
         master.safe_psql('postgres', 'CLUSTER t_heap USING t_gist')
 
-        if master.major_version < 12:
+        if master.major_version < 11:
             master.safe_psql('postgres', 'CHECKPOINT')
 
         # Sync master and replica
         self.wait_until_replica_catch_with_master(master, replica)
 
-        if master.major_version < 12:
+        if master.major_version < 11:
             replica.safe_psql('postgres', 'CHECKPOINT')
             self.check_ptrack_map_sanity(replica, idx_ptrack)
 
@@ -2660,15 +2844,14 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
             set_replication=True,
             initdb_params=['--data-checksums'],
             ptrack_enable=True,
-            pg_options={
+            pg_options={'autovacuum': 'off'})
-                'autovacuum': 'off'})
 
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
         master.slow_start()
 
-        if master.major_version >= 12:
+        if master.major_version >= 11:
             master.safe_psql(
                 "postgres",
                 "CREATE EXTENSION ptrack")
@@ -2777,7 +2960,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
             "md5(repeat(i::text,10))::tsvector as tsvector "
             "from generate_series(0,2560) i")
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
                     node.safe_psql(
@@ -2793,7 +2976,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'truncate t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 # get fork size and calculate it in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@@ -2810,7 +2993,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         pgdata = self.pgdata_content(node.data_dir)
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
                     node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
@@ -2840,14 +3023,15 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
             pg_options={
                 'max_wal_size': '32MB',
                 'archive_timeout': '10s',
-                'checkpoint_timeout': '30s'})
+                'checkpoint_timeout': '30s',
+                'autovacuum': 'off'})
 
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'master', master)
         master.slow_start()
 
-        if master.major_version >= 12:
+        if master.major_version >= 11:
             master.safe_psql(
                 "postgres",
                 "CREATE EXTENSION ptrack")
@@ -2885,7 +3069,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
-        if replica.major_version < 12:
+        if replica.major_version < 11:
             for i in idx_ptrack:
                 # get fork size and calculate it in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -2905,7 +3089,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
                 '--master-db=postgres',
                 '--master-port={0}'.format(master.port)])
 
-        if replica.major_version < 12:
+        if replica.major_version < 11:
             for i in idx_ptrack:
                 idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
                     replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
@@ -2915,7 +3099,6 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
 
         # Sync master and replica
         self.wait_until_replica_catch_with_master(master, replica)
-        replica.safe_psql('postgres', 'checkpoint')
 
         self.backup_node(
             backup_dir, 'replica', replica, backup_type='ptrack',
@@ -2987,7 +3170,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.backup_node(
             backup_dir, 'node', node, options=['-j10', '--stream'])
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 # get fork size and calculate it in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@@ -3006,7 +3189,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY
-        if node.major_version < 12:
+        if node.major_version < 11:
             self.check_ptrack_map_sanity(node, idx_ptrack)
 
         self.backup_node(
@@ -3044,7 +3227,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'master', master)
         master.slow_start()
 
-        if master.major_version >= 12:
+        if master.major_version >= 11:
             master.safe_psql(
                 "postgres",
                 "CREATE EXTENSION ptrack")
@@ -3092,7 +3275,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
                 '--master-port={0}'.format(master.port),
                 '--stream'])
 
-        if replica.major_version < 12:
+        if replica.major_version < 11:
             for i in idx_ptrack:
                 # get fork size and calculate it in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -3115,7 +3298,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY
-        if replica.major_version < 12:
+        if replica.major_version < 11:
             self.check_ptrack_map_sanity(master, idx_ptrack)
 
         self.backup_node(
@@ -3184,7 +3367,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'vacuum freeze t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 # get size of heap and indexes. size calculated in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@@ -3195,7 +3378,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
                     idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
 
         # CHECK PTRACK SANITY
-        if node.major_version < 12:
+        if node.major_version < 11:
             self.check_ptrack_map_sanity(node, idx_ptrack)
 
         self.backup_node(
@@ -3230,7 +3413,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'master', master)
         master.slow_start()
 
-        if master.major_version >= 12:
+        if master.major_version >= 11:
             master.safe_psql(
                 "postgres",
                 "CREATE EXTENSION ptrack")
@@ -3279,7 +3462,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
                 '--master-port={0}'.format(master.port),
                 '--stream'])
 
-        if replica.major_version < 12:
+        if replica.major_version < 11:
             for i in idx_ptrack:
                 # get size of heap and indexes. size calculated in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -3297,7 +3480,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         replica.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY
-        if replica.major_version < 12:
+        if replica.major_version < 11:
             self.check_ptrack_map_sanity(master, idx_ptrack)
 
         self.backup_node(
@@ -3360,7 +3543,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.backup_node(
             backup_dir, 'node', node, options=['-j10', '--stream'])
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 # get size of heap and indexes. size calculated in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@@ -3374,7 +3557,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'checkpoint')
 
         # CHECK PTRACK SANITY
-        if node.major_version < 12:
+        if node.major_version < 11:
             self.check_ptrack_map_sanity(node, idx_ptrack)
 
         self.backup_node(
@@ -3438,7 +3621,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.backup_node(
             backup_dir, 'node', node, options=['-j10', '--stream'])
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 # get size of heap and indexes. size calculated in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@@ -3452,7 +3635,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'vacuum full t_heap')
         node.safe_psql('postgres', 'checkpoint')
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             self.check_ptrack_map_sanity(node, idx_ptrack)
 
         self.backup_node(
@@ -3490,7 +3673,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'master', master)
         master.slow_start()
 
-        if master.major_version >= 12:
+        if master.major_version >= 11:
             master.safe_psql(
                 "postgres",
                 "CREATE EXTENSION ptrack")
@@ -3514,7 +3697,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
             "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as "
             "tsvector from generate_series(0,256000) i")
 
-        if master.major_version < 12:
+        if master.major_version < 11:
             for i in idx_ptrack:
                 if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
                     master.safe_psql(
@@ -3541,7 +3724,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
                 '--master-port={0}'.format(master.port),
                 '--stream'])
 
-        if replica.major_version < 12:
+        if replica.major_version < 11:
             for i in idx_ptrack:
                 # get size of heap and indexes. size calculated in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -3559,7 +3742,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.wait_until_replica_catch_with_master(master, replica)
         replica.safe_psql('postgres', 'checkpoint')
 
-        if replica.major_version < 12:
+        if replica.major_version < 11:
             self.check_ptrack_map_sanity(master, idx_ptrack)
 
         self.backup_node(
@@ -3606,7 +3789,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
             "md5(repeat(i::text,10))::tsvector as tsvector "
             "from generate_series(0,2560) i")
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
                     node.safe_psql(
@@ -3619,7 +3802,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.backup_node(
             backup_dir, 'node', node, options=['-j10', '--stream'])
 
-        if node.major_version < 12:
+        if node.major_version < 11:
             for i in idx_ptrack:
                 # get size of heap and indexes. size calculated in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@@ -3634,7 +3817,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         node.safe_psql('postgres', 'CHECKPOINT')
 
         # CHECK PTRACK SANITY
-        if node.major_version < 12:
+        if node.major_version < 11:
             self.check_ptrack_map_sanity(node, idx_ptrack)
 
         self.backup_node(
@@ -3670,7 +3853,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         self.add_instance(backup_dir, 'master', master)
         master.slow_start()
 
-        if master.major_version >= 12:
+        if master.major_version >= 11:
             master.safe_psql(
                 "postgres",
                 "CREATE EXTENSION ptrack")
@@ -3718,7 +3901,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
                 ]
             )
 
-        if master.major_version < 12:
+        if master.major_version < 11:
             for i in idx_ptrack:
                 # get size of heap and indexes. size calculated in pages
                 idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -3737,7 +3920,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
         replica.safe_psql('postgres', 'CHECKPOINT')
 
         # CHECK PTRACK SANITY
-        if master.major_version < 12:
+        if master.major_version < 11:
             self.check_ptrack_map_sanity(master, idx_ptrack)
 
         self.backup_node(