diff --git a/tests/external.py b/tests/external.py index 77e24267..dfc8fc93 100644 --- a/tests/external.py +++ b/tests/external.py @@ -76,9 +76,8 @@ class ExternalTest(ProbackupTest, unittest.TestCase): options=[ '--external-dirs={0}'.format(external_dir)]) - if self.paranoia: - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) node.cleanup() shutil.rmtree(external_dir, ignore_errors=True) @@ -86,10 +85,9 @@ class ExternalTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=["-j", "4"]) - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -316,7 +314,12 @@ class ExternalTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility(self): - """""" + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + restore delta backup, check that incremental chain + restored correctly + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -357,15 +360,15 @@ class ExternalTest(ProbackupTest, unittest.TestCase): pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) - # FULL backup backup_id = self.backup_node( backup_dir, 'node', node, old_binary=True, options=["-j", "4", "--stream"]) # fill external directories with changed data + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + self.restore_node( backup_dir, 'node', node, data_dir=external_dir1_old, options=["-j", "4"]) @@ -412,6 +415,222 @@ class ExternalTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_backward_compatibility_merge_1(self): + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + merge delta backup ajd restore it + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + node.slow_start() + + node.pgbench_init(scale=5) + + # tmp FULL backup with old binary + tmp_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # delta backup with external directories using new binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge chain chain with new binary + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_backward_compatibility_merge_2(self): + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + merge delta backup ajd restore it + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + node.slow_start() + + node.pgbench_init(scale=5) + + # tmp FULL backup with old binary + tmp_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # delta backup with external directories using new binary + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # Fill external dirs with changed data + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, + options=['-j', '4', '--skip-external-dirs']) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, + options=['-j', '4', '--skip-external-dirs']) + + # delta backup without external directories using old binary + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge chain chain with new binary + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge(self): @@ -1012,7 +1231,7 @@ class ExternalTest(ProbackupTest, unittest.TestCase): # latest page backup without external_dir + # multiple external directories + # --external-dirs=none + - # --external-dirs point to a file + # --external-dirs point to a file + # external directory in config and in command line + # external directory contain multuple directories, some of them my be empty + # forbid to external-dirs to point to tablespace directories