diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 7cfef734..ed747cd4 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -29,6 +29,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup backup -B backup-path -b backup-mode --instance=instance_name [-C] [--stream [-S slot-name]] [--backup-pg-log] [-j num-threads] [--archive-timeout=archive-timeout] + [--compress] [--compress-algorithm=compress-algorithm] [--compress-level=compress-level] [--progress] [--delete-expired] diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 4bdddce0..18b0da77 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -815,11 +815,14 @@ class ProbackupTest(object): """ Returns current user name """ return pwd.getpwuid(os.getuid())[0] + def version_to_num(self, version): + return testgres.version_to_num(version) + def switch_wal_segment(self, node): """ Execute pg_switch_wal/xlog() in given node""" - if testgres.version_to_num( + if self.version_to_num( node.safe_psql("postgres", "show server_version") - ) >= testgres.version_to_num('10.0'): + ) >= self.version_to_num('10.0'): node.safe_psql("postgres", "select pg_switch_wal()") else: node.safe_psql("postgres", "select pg_switch_xlog()") diff --git a/tests/pgpro589.py b/tests/pgpro589.py index ebcaa7cd..a67f3dd4 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -18,7 +18,8 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): check that no files where copied to backup catalogue """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -39,25 +40,41 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): ) pgbench.wait() pgbench.stdout.close() - path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip().decode("utf-8") + path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").rstrip().decode( + "utf-8") try: - self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10']) + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10']) # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of missing archive wal segment with start_lsn.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of missing archive wal " + "segment with start_lsn.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: wait for WAL segment' in e.message - and 'ERROR: switched WAL segment' in e.message - and 'could not be archived' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Wait for WAL segment' in e.message and + 'ERROR: Switched WAL segment' in e.message and + 'could not be archived' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) backup_id = self.show_pb(backup_dir, 'node')[0]['ID'] - self.assertEqual('ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup should have ERROR status') - file = os.path.join(backup_dir, 'backups', 'node', backup_id, 'database', path) - self.assertFalse(os.path.isfile(file), - '\n Start LSN was not found in archive but datafiles where copied to backup catalogue.\n For example: {0}\n It is not optimal'.format(file)) + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup should have ERROR status') + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', path) + self.assertFalse( + os.path.isfile(file), + "\n Start LSN was not found in archive but datafiles where " + "copied to backup catalogue.\n For example: {0}\n " + "It is not optimal".format(file)) # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/retention_test.py b/tests/retention_test.py index 152c2ac2..ad3766ca 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -71,7 +71,12 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.start() - with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf: + with open( + os.path.join( + backup_dir, + 'backups', + 'node', + "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy = 1\n") conf.write("retention-window = 1\n") @@ -86,7 +91,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase): for backup in os.listdir(backups): if backup == 'pg_probackup.conf': continue - with open(os.path.join(backups, backup, "backup.control"), "a") as conf: + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=days_delta))) days_delta -= 1 @@ -107,7 +114,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_wal(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -119,13 +127,17 @@ class RetentionTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100500) i") + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") # Take FULL BACKUP self.backup_node(backup_dir, 'node', node) node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100500) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") self.backup_node(backup_dir, 'node', node) @@ -134,7 +146,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase): for backup in os.listdir(backups): if backup == 'pg_probackup.conf': continue - with open(os.path.join(backups, backup, "backup.control"), "a") as conf: + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=days_delta))) days_delta -= 1 @@ -142,10 +156,11 @@ class RetentionTest(ProbackupTest, unittest.TestCase): # Make backup to be keeped self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) # Purge backups - self.delete_expired(backup_dir, 'node') + self.delete_expired( + backup_dir, 'node', options=['--retention-window=2']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) # Clean after yourself diff --git a/tests/validate_test.py b/tests/validate_test.py index 3f74e0ce..3a4984c6 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -116,7 +116,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_validate_corrupted_intermediate_backup(self): """make archive node, take FULL, PAGE1, PAGE2 backups, corrupt file in PAGE1 backup, - run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 get status INVALID""" + run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" fname = self.id().split('.')[3] node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], @@ -261,7 +261,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): corrupt file in PAGE1 and PAGE4, run validate on PAGE3, expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -277,57 +278,77 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # PAGE1 node.safe_psql( "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") - backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # PAGE2 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") file_page_2 = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").rstrip() - backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # PAGE3 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") - backup_id_4 = self.backup_node(backup_dir, 'node', node, backup_type='page') + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + backup_id_4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # PAGE4 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") - backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') - # PAGE5 + # PAGE5 node.safe_psql( "postgres", - "create table t_heap1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") file_page_5 = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap1')").rstrip() - backup_id_6 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_6 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # PAGE6 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") - backup_id_7 = self.backup_node(backup_dir, 'node', node, backup_type='page') + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_7 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # FULL2 backup_id_8 = self.backup_node(backup_dir, 'node', node) # Corrupt some file in PAGE2 and PAGE5 backups - file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + file_page1 = os.path.join( + backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) with open(file_page1, "rb+", 0) as f: f.seek(84) f.write(b"blah") f.flush() f.close - file_page4 = os.path.join(backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + file_page4 = os.path.join( + backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) with open(file_page4, "rb+", 0) as f: f.seek(42) f.write(b"blah") @@ -336,51 +357,97 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Validate PAGE3 try: - self.validate_pb(backup_dir, 'node', backup_id=backup_id_4, + self.validate_pb( + backup_dir, 'node', + backup_id=backup_id_4, options=['--log-level-file=verbose']) - self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Validating parents for backup {0}'.format(backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating parents for backup {0}'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_1) in e.message - and 'INFO: Backup {0} data files are valid'.format(backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_2) in e.message - and 'INFO: Backup {0} data files are valid'.format(backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_3) in e.message - and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_page1) in e.message - and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_page1) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_4, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_5, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_6, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_7, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format(repr(e.message), self.cmd)) - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], + 'Backup STATUS should be "OK"') # Clean after yourself # self.del_test_dir(module_name, fname) @@ -521,7 +588,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): corrupt file in PAGE1 backup and run validate on instance, expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -533,31 +601,43 @@ class ValidateTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) node.safe_psql( "postgres", - "create table t_heap1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") file_path_t_heap1 = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap1')").rstrip() # PAGE1 - backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") # PAGE2 - backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # FULL1 - backup_id_4 = self.backup_node(backup_dir, 'node', node) + backup_id_4 = self.backup_node( + backup_dir, 'node', node) # PAGE3 - backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path_t_heap1) + file_full = os.path.join( + backup_dir, 'backups/node', backup_id_2, + 'database', file_path_t_heap1) with open(file_full, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -566,48 +646,86 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Validate Instance try: - self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) - self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.validate_pb( + backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( "INFO: Validate backups of the instance 'node'" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_5) in e.message - and 'INFO: Backup {0} data files are valid'.format(backup_id_5) in e.message - and 'INFO: Backup {0} WAL segments are valid'.format(backup_id_5) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating backup {0}'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_5) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_4) in e.message - and 'INFO: Backup {0} data files are valid'.format(backup_id_4) in e.message - and 'INFO: Backup {0} WAL segments are valid'.format(backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating backup {0}'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_3) in e.message - and 'INFO: Backup {0} data files are valid'.format(backup_id_3) in e.message - and 'INFO: Backup {0} WAL segments are valid'.format(backup_id_3) in e.message - and 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_3, backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_3) in e.message and + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_3, backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_2) in e.message - and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message - and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_full) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_1) in e.message - and 'INFO: Backup {0} data files are valid'.format(backup_id_1) in e.message - and 'INFO: Backup {0} WAL segments are valid'.format(backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertTrue( 'INFO: Some backups are not valid' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') # Clean after yourself # self.del_test_dir(module_name, fname) @@ -692,7 +810,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): def test_validate_corrupt_wal_1(self): """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -724,14 +843,28 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Simple validate try: self.validate_pb(backup_dir, 'node') - self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue('Possible WAL CORRUPTION' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') # Clean after yourself self.del_test_dir(module_name, fname) @@ -836,30 +969,155 @@ class ValidateTest(ProbackupTest, unittest.TestCase): wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] file = os.path.join(backup_dir, 'wal', 'node', wals[1]) os.remove(file) + + # cut out '.gz' + if self.archive_compress: + file = file[:-3] + try: self.validate_pb(backup_dir, 'node') - self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn('WARNING: WAL segment "{0}" is absent'.format(file), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.assertIn('WARNING: There are not enough WAL records to consistenly restore backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.assertIn('WARNING: Backup {0} WAL segments are corrupted'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.assertIn('INFO: Some backups are not valid', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + "WARNING: WAL segment \"{0}\" is absent".format( + file) in e.message and + "WARNING: There are not enough WAL records to consistenly " + "restore backup {0}".format(backup_id) in e.message and + "WARNING: Backup {0} WAL segments are corrupted".format( + backup_id) in e.message and + "INFO: Some backups are not valid" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"') + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup {0} should have STATUS "CORRUPT"') # Be paranoid and run validate again try: self.validate_pb(backup_dir, 'node') - self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertIn( + 'INFO: Backup {0} has status CORRUPT. ' + 'Skip validation.\n'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_between_backups(self): + """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + if self.get_version(node) < self.version_to_num('10.0'): + walfile = node.safe_psql( + 'postgres', + 'select pg_xlogfile_name(pg_current_xlog_location())').rstrip() + else: + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_location())').rstrip() + + if self.archive_compress: + walfile = walfile + '.gz' + self.switch_wal_segment(node) + + # generate some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: + f.seek(9000) + f.write(b"b") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: not enough WAL records to xid' in e.message and + 'WARNING: recovery can be done up to time' in e.message and + "ERROR: not enough WAL records to xid {0}\n".format( + target_xid), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['Status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['Status'], + 'Backup STATUS should be "OK"') # Clean after yourself self.del_test_dir(module_name, fname)