From 6dcf64a90705411e2764800ff8dd2ccb5fc1a4d3 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 10 Feb 2021 16:22:40 +0300 Subject: [PATCH 1/6] Tests. Fix cfs_backup tests compatibility with ptrack 2.1 --- tests/cfs_backup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 5a366551..3d2fa8de 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -20,9 +20,9 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, self.fname), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'cfs_encryption': 'off', 'max_wal_senders': '2', 'shared_buffers': '200MB' @@ -35,6 +35,11 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.node.slow_start() + if self.node.major_version >= 12: + self.node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) tblspace = self.node.safe_psql( From 2305d1fdda20fc16af053a06805416de0d2e69de Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 10 Feb 2021 19:44:38 +0300 Subject: [PATCH 2/6] Tests. Expect failure in test_validate_target_lsn, because it requires multi-timeline WAL validation, which is not implemented yet --- tests/validate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/validate.py b/tests/validate.py index 6777b92a..cfec234d 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3451,7 +3451,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.expectedFailure + #TODO fix the test + @unittest.expectedFailure # @unittest.skip("skip") def test_validate_target_lsn(self): """ From 5207900fb843572398513998b6ec158d74473d23 Mon Sep 17 00:00:00 2001 From: anastasia Date: Thu, 11 Feb 2021 12:01:13 +0300 Subject: [PATCH 3/6] Update tests/Readme.md. Add note about python3. --- tests/Readme.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index 3adf0c01..adcf5380 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -1,7 +1,11 @@ -[см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) +[see wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) ``` -Note: For now these are works on Linux and "kinda" works on Windows +Note: For now these tests work on Linux and "kinda" work on Windows +``` + +``` +Note: tests require python3 to work properly. ``` ``` From 71661abc3f4930d5d960f85dda0deae71a31fc63 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Feb 2021 19:28:42 +0300 Subject: [PATCH 4/6] tests: fix "cfs_backup" module --- tests/cfs_backup.py | 76 ++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 3d2fa8de..b7ebff0f 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -45,13 +45,17 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( - tblspace_name) - ) - self.assertTrue( - tblspace_name in tblspace and "compression=true" in tblspace, + tblspace_name)) + + self.assertIn( + tblspace_name, str(tblspace), "ERROR: The tablespace not created " - "or it create without compressions" - ) + "or it create without compressions") + + self.assertIn( + "compression=true", str(tblspace), + "ERROR: The tablespace not created " + "or it create without compressions") self.assertTrue( find_by_name( @@ -743,12 +747,14 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): # CHECK FULL BACKUP self.node.stop() self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_full, options=["-j", "4"]) + self.backup_dir, 'node', self.node, backup_id=backup_id_full, + options=[ + "-j", "4", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + self.node.slow_start() self.assertEqual( full_result, @@ -762,8 +768,12 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.get_tblspace_path(self.node, tblspace_name), ignore_errors=True) self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_page, options=["-j", "4"]) + self.backup_dir, 'node', self.node, backup_id=backup_id_page, + options=[ + "-j", "4", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + self.node.slow_start() self.assertEqual( page_result, @@ -791,8 +801,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): "AS SELECT i AS id, MD5(i::text) AS text, " "MD5(repeat(i::text,10))::tsvector AS tsvector " "FROM generate_series(0,1005000) i".format( - 't_heap_1', tblspace_name_1) - ) + 't_heap_1', tblspace_name_1)) self.node.safe_psql( "postgres", @@ -800,8 +809,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): "AS SELECT i AS id, MD5(i::text) AS text, " "MD5(repeat(i::text,10))::tsvector AS tsvector " "FROM generate_series(0,1005000) i".format( - 't_heap_2', tblspace_name_2) - ) + 't_heap_2', tblspace_name_2)) full_result_1 = self.node.safe_psql( "postgres", "SELECT * FROM t_heap_1") @@ -869,21 +877,16 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): # CHECK FULL BACKUP self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_1), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_2), - ignore_errors=True) self.restore_node( self.backup_dir, 'node', self.node, - backup_id=backup_id_full, options=["-j", "4"]) + backup_id=backup_id_full, + options=[ + "-j", "4", "--incremental-mode=checksum", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) self.node.slow_start() + self.assertEqual( full_result_1, self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), @@ -895,21 +898,16 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): # CHECK PAGE BACKUP self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_1), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_2), - ignore_errors=True) self.restore_node( self.backup_dir, 'node', self.node, - backup_id=backup_id_page, options=["-j", "4"]) + backup_id=backup_id_page, + options=[ + "-j", "4", "--incremental-mode=checksum", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) self.node.slow_start() + self.assertEqual( page_result_1, self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), From f324b081809b2f27c28f66b1cf2f878f55131c25 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Feb 2021 20:27:47 +0300 Subject: [PATCH 5/6] tests: more fixes for "cfs_backup" module --- tests/cfs_backup.py | 8 ++++---- tests/helpers/cfs_helpers.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index b7ebff0f..2e686d46 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -482,7 +482,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): ) # --- Section: Incremental from fill tablespace --- # - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_ptrack_after_create_table(self): @@ -546,7 +546,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): ) ) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): @@ -612,7 +612,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): ) ) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_page_after_create_table(self): @@ -917,7 +917,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), 'Lost data after restore') - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_page_after_create_table_stream(self): diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py index 67e2b331..31af76f2 100644 --- a/tests/helpers/cfs_helpers.py +++ b/tests/helpers/cfs_helpers.py @@ -88,4 +88,6 @@ def corrupt_file(filename): def random_string(n): a = string.ascii_letters + string.digits - return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file + random_str = ''.join([random.choice(a) for i in range(int(n)+1)]) + return str.encode(random_str) +# return ''.join([random.choice(a) for i in range(int(n)+1)]) From 31a8ea3ff04c226a12ed93db3aa82b4fd959367b Mon Sep 17 00:00:00 2001 From: anastasia Date: Fri, 12 Feb 2021 13:08:12 +0300 Subject: [PATCH 6/6] tests: Skip test_incr_lsn_long_xact_1, when testing with PostgresPro Enterprise --- tests/incr_restore.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 1d3a52df..a228bdd7 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -1711,6 +1711,9 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure + # This test will pass with Enterprise + # because it has checksums enabled by default + @unittest.skipIf(ProbackupTest.enterprise, 'skip') def test_incr_lsn_long_xact_1(self): """ """