From 3475ef663114d68152ef00c6a8b3a18b4dbdf5f9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Jun 2020 13:25:22 +0300 Subject: [PATCH 1/3] [PBCKP-98] increase PAGE_READ_ATTEMPTS up to 300 --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3d773e4d..3122293f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -88,7 +88,7 @@ extern const char *PROGRAM_EMAIL; #define STDIO_BUFSIZE 65536 /* retry attempts */ -#define PAGE_READ_ATTEMPTS 100 +#define PAGE_READ_ATTEMPTS 300 /* max size of note, that can be added to backup */ #define MAX_NOTE_SIZE 1024 From 6f89a53ce011f0f7e9e7ac38ddfea110972616ca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jun 2020 10:39:29 +0300 Subject: [PATCH 2/3] tests: for basic smoke tests avoid crashish of PostgreSQL cluster when cleaning up test directories --- tests/archive.py | 2 +- tests/backup.py | 8 ++++---- tests/checkdb.py | 2 +- tests/compression.py | 3 +-- tests/delta.py | 2 +- tests/helpers/ptrack_helpers.py | 5 ++++- tests/merge.py | 3 +-- tests/page.py | 2 +- tests/replica.py | 2 +- tests/retention.py | 6 +++--- tests/validate.py | 2 +- 11 files changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 5508f54b..444b3dd0 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -988,7 +988,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'master', replica) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, nodes=[master, replica]) # @unittest.expectedFailure # @unittest.skip("skip") diff --git a/tests/backup.py b/tests/backup.py index 694c9a44..f4712ad7 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1068,7 +1068,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, nodes=[node]) # @unittest.skip("skip") def test_tablespace_handling_1(self): @@ -1646,7 +1646,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.assertEqual(show_backup['status'], "OK") # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, nodes=[node]) # @unittest.skip("skip") def test_pg_11_adjusted_wal_segment_size(self): @@ -1930,7 +1930,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): os.chmod(full_path, 700) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_basic_missing_dir_permissions(self): @@ -1973,7 +1973,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): os.chmod(full_path, 700) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_backup_with_least_privileges_role(self): diff --git a/tests/checkdb.py b/tests/checkdb.py index 033a6c25..6c25293a 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -348,7 +348,7 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): log_file_content) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_checkdb_block_validation_sanity(self): diff --git a/tests/compression.py b/tests/compression.py index b8788a46..321461d6 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -115,10 +115,9 @@ class CompressionTest(ProbackupTest, unittest.TestCase): delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(delta_result, delta_result_new) - node.cleanup() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) def test_compression_archive_zlib(self): """ diff --git a/tests/delta.py b/tests/delta.py index fdbaf127..6b2ebf92 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -87,7 +87,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored.slow_start() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_delta_vacuum_truncate_1(self): diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index eec57a3b..bba6cf2a 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1448,13 +1448,16 @@ class ProbackupTest(object): def get_bin_path(self, binary): return testgres.get_bin_path(binary) - def del_test_dir(self, module_name, fname): + def del_test_dir(self, module_name, fname, nodes=[]): """ Del testdir and optimistically try to del module dir""" try: testgres.clean_all() except: pass + for node in nodes: + node.stop() + shutil.rmtree( os.path.join( self.tmp_path, diff --git a/tests/merge.py b/tests/merge.py index 54b7e288..93d41c5d 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -100,8 +100,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): self.assertEqual(count1, count2) # Clean after yourself - node.cleanup() - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) def test_merge_compressed_backups(self): """ diff --git a/tests/page.py b/tests/page.py index a73328e1..201f825e 100644 --- a/tests/page.py +++ b/tests/page.py @@ -100,7 +100,7 @@ class PageTest(ProbackupTest, unittest.TestCase): self.assertEqual(result1, result2) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node, node_restored]) # @unittest.skip("skip") def test_page_vacuum_truncate_1(self): diff --git a/tests/replica.py b/tests/replica.py index ab6eaf59..aa3de9ae 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -344,7 +344,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): options=['--archive-timeout=30s', '--stream']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [master, replica]) # @unittest.skip("skip") def test_take_backup_from_delayed_replica(self): diff --git a/tests/retention.py b/tests/retention.py index e797d3c6..876bcbca 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1009,7 +1009,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): 'FULL') # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants_1(self): @@ -1276,7 +1276,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): '--delete-expired', '--log-level-console=log']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_window_chains(self): @@ -2534,4 +2534,4 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) diff --git a/tests/validate.py b/tests/validate.py index 8e27c50c..c2d1405b 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -298,7 +298,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): 'Backup STATUS should be "ORPHAN"') # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups(self): From 615a54edb56d88f119cb81c764c91f8e636102a8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jun 2020 10:49:32 +0300 Subject: [PATCH 3/3] tests: minor fixes --- tests/backup.py | 2 +- tests/delta.py | 23 ++++++++--------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index f4712ad7..fadc2aa5 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1601,7 +1601,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): options=['--stream', '--slot=slot_1', '--temp-slot']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_backup_concurrent_drop_table(self): diff --git a/tests/delta.py b/tests/delta.py index 6b2ebf92..0abdd1c2 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -45,39 +45,32 @@ class DeltaTest(ProbackupTest, unittest.TestCase): "create table t_heap as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;" - ) + "from generate_series(0,1024) i;") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node(backup_dir, 'node', node, options=['--stream']) node.safe_psql( "postgres", - "delete from t_heap where ctid >= '(11,0)'" - ) + "delete from t_heap where ctid >= '(11,0)'") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) self.restore_node( - backup_dir, 'node', node_restored - ) + backup_dir, 'node', node_restored) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -87,7 +80,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored.slow_start() # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname, [node, node_restored]) # @unittest.skip("skip") def test_delta_vacuum_truncate_1(self):