mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2025-02-08 14:28:36 +02:00
tests: major fixes
This commit is contained in:
Grigory Smolkin
parent
45923cf9c7
commit
68e2742da0
@ -29,6 +29,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
|
||||
pg_probackup backup -B backup-path -b backup-mode --instance=instance_name
|
||||
[-C] [--stream [-S slot-name]] [--backup-pg-log]
|
||||
[-j num-threads] [--archive-timeout=archive-timeout]
|
||||
[--compress]
|
||||
[--compress-algorithm=compress-algorithm]
|
||||
[--compress-level=compress-level]
|
||||
[--progress] [--delete-expired]
|
||||
|
||||
@ -815,11 +815,14 @@ class ProbackupTest(object):
|
||||
""" Returns current user name """
|
||||
return pwd.getpwuid(os.getuid())[0]
|
||||
|
||||
def version_to_num(self, version):
|
||||
return testgres.version_to_num(version)
|
||||
|
||||
def switch_wal_segment(self, node):
|
||||
""" Execute pg_switch_wal/xlog() in given node"""
|
||||
if testgres.version_to_num(
|
||||
if self.version_to_num(
|
||||
node.safe_psql("postgres", "show server_version")
|
||||
) >= testgres.version_to_num('10.0'):
|
||||
) >= self.version_to_num('10.0'):
|
||||
node.safe_psql("postgres", "select pg_switch_wal()")
|
||||
else:
|
||||
node.safe_psql("postgres", "select pg_switch_xlog()")
|
||||
|
||||
@ -18,7 +18,8 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase):
|
||||
check that no files where copied to backup catalogue
|
||||
"""
|
||||
fname = self.id().split('.')[3]
|
||||
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
node = self.make_simple_node(
|
||||
base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
initdb_params=['--data-checksums'],
|
||||
pg_options={'wal_level': 'replica'}
|
||||
)
|
||||
@ -39,25 +40,41 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase):
|
||||
)
|
||||
pgbench.wait()
|
||||
pgbench.stdout.close()
|
||||
path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip().decode("utf-8")
|
||||
path = node.safe_psql(
|
||||
"postgres",
|
||||
"select pg_relation_filepath('pgbench_accounts')").rstrip().decode(
|
||||
"utf-8")
|
||||
|
||||
try:
|
||||
self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10'])
|
||||
self.backup_node(
|
||||
backup_dir, 'node', node,
|
||||
options=['--archive-timeout=10'])
|
||||
# we should die here because exception is what we expect to happen
|
||||
self.assertEqual(1, 0, "Expecting Error because of missing archive wal segment with start_lsn.\n Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of missing archive wal "
|
||||
"segment with start_lsn.\n Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertTrue(
|
||||
'INFO: wait for WAL segment' in e.message
|
||||
and 'ERROR: switched WAL segment' in e.message
|
||||
and 'could not be archived' in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Wait for WAL segment' in e.message and
|
||||
'ERROR: Switched WAL segment' in e.message and
|
||||
'could not be archived' in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
backup_id = self.show_pb(backup_dir, 'node')[0]['ID']
|
||||
self.assertEqual('ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup should have ERROR status')
|
||||
file = os.path.join(backup_dir, 'backups', 'node', backup_id, 'database', path)
|
||||
self.assertFalse(os.path.isfile(file),
|
||||
'\n Start LSN was not found in archive but datafiles where copied to backup catalogue.\n For example: {0}\n It is not optimal'.format(file))
|
||||
self.assertEqual(
|
||||
'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'],
|
||||
'Backup should have ERROR status')
|
||||
file = os.path.join(
|
||||
backup_dir, 'backups', 'node',
|
||||
backup_id, 'database', path)
|
||||
self.assertFalse(
|
||||
os.path.isfile(file),
|
||||
"\n Start LSN was not found in archive but datafiles where "
|
||||
"copied to backup catalogue.\n For example: {0}\n "
|
||||
"It is not optimal".format(file))
|
||||
|
||||
# Clean after yourself
|
||||
self.del_test_dir(module_name, fname)
|
||||
|
||||
@ -71,7 +71,12 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
self.set_archiving(backup_dir, 'node', node)
|
||||
node.start()
|
||||
|
||||
with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf:
|
||||
with open(
|
||||
os.path.join(
|
||||
backup_dir,
|
||||
'backups',
|
||||
'node',
|
||||
"pg_probackup.conf"), "a") as conf:
|
||||
conf.write("retention-redundancy = 1\n")
|
||||
conf.write("retention-window = 1\n")
|
||||
|
||||
@ -86,7 +91,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
for backup in os.listdir(backups):
|
||||
if backup == 'pg_probackup.conf':
|
||||
continue
|
||||
with open(os.path.join(backups, backup, "backup.control"), "a") as conf:
|
||||
with open(
|
||||
os.path.join(
|
||||
backups, backup, "backup.control"), "a") as conf:
|
||||
conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format(
|
||||
datetime.now() - timedelta(days=days_delta)))
|
||||
days_delta -= 1
|
||||
@ -107,7 +114,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
def test_retention_wal(self):
|
||||
"""purge backups using window-based retention policy"""
|
||||
fname = self.id().split('.')[3]
|
||||
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
node = self.make_simple_node(
|
||||
base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
initdb_params=['--data-checksums'],
|
||||
pg_options={'wal_level': 'replica'}
|
||||
)
|
||||
@ -119,13 +127,17 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100500) i")
|
||||
"create table t_heap as select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(0,100500) i")
|
||||
|
||||
# Take FULL BACKUP
|
||||
self.backup_node(backup_dir, 'node', node)
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100500) i")
|
||||
"insert into t_heap select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(0,100500) i")
|
||||
|
||||
self.backup_node(backup_dir, 'node', node)
|
||||
|
||||
@ -134,7 +146,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
for backup in os.listdir(backups):
|
||||
if backup == 'pg_probackup.conf':
|
||||
continue
|
||||
with open(os.path.join(backups, backup, "backup.control"), "a") as conf:
|
||||
with open(
|
||||
os.path.join(
|
||||
backups, backup, "backup.control"), "a") as conf:
|
||||
conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format(
|
||||
datetime.now() - timedelta(days=days_delta)))
|
||||
days_delta -= 1
|
||||
@ -142,10 +156,11 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
# Make backup to be keeped
|
||||
self.backup_node(backup_dir, 'node', node, backup_type="page")
|
||||
|
||||
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4)
|
||||
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3)
|
||||
|
||||
# Purge backups
|
||||
self.delete_expired(backup_dir, 'node')
|
||||
self.delete_expired(
|
||||
backup_dir, 'node', options=['--retention-window=2'])
|
||||
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2)
|
||||
|
||||
# Clean after yourself
|
||||
|
||||
@ -116,7 +116,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
# @unittest.skip("skip")
|
||||
def test_validate_corrupted_intermediate_backup(self):
|
||||
"""make archive node, take FULL, PAGE1, PAGE2 backups, corrupt file in PAGE1 backup,
|
||||
run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 get status INVALID"""
|
||||
run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN"""
|
||||
fname = self.id().split('.')[3]
|
||||
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
initdb_params=['--data-checksums'],
|
||||
@ -261,7 +261,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
corrupt file in PAGE1 and PAGE4, run validate on PAGE3,
|
||||
expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN"""
|
||||
fname = self.id().split('.')[3]
|
||||
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
node = self.make_simple_node(
|
||||
base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
initdb_params=['--data-checksums'],
|
||||
pg_options={'wal_level': 'replica'}
|
||||
)
|
||||
@ -277,57 +278,77 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
# PAGE1
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i")
|
||||
backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
"create table t_heap as select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(0,10000) i")
|
||||
backup_id_2 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
# PAGE2
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i")
|
||||
"insert into t_heap select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(0,10000) i")
|
||||
file_page_2 = node.safe_psql(
|
||||
"postgres",
|
||||
"select pg_relation_filepath('t_heap')").rstrip()
|
||||
backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
backup_id_3 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
# PAGE3
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i")
|
||||
backup_id_4 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
"insert into t_heap select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(10000,20000) i")
|
||||
backup_id_4 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
# PAGE4
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i")
|
||||
backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
"insert into t_heap select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(20000,30000) i")
|
||||
backup_id_5 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
# PAGE5
|
||||
# PAGE5
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"create table t_heap1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i")
|
||||
"create table t_heap1 as select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(0,10000) i")
|
||||
file_page_5 = node.safe_psql(
|
||||
"postgres",
|
||||
"select pg_relation_filepath('t_heap1')").rstrip()
|
||||
backup_id_6 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
backup_id_6 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
# PAGE6
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i")
|
||||
backup_id_7 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
"insert into t_heap select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(30000,40000) i")
|
||||
backup_id_7 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
# FULL2
|
||||
backup_id_8 = self.backup_node(backup_dir, 'node', node)
|
||||
|
||||
# Corrupt some file in PAGE2 and PAGE5 backups
|
||||
file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_3, 'database', file_page_2)
|
||||
file_page1 = os.path.join(
|
||||
backup_dir, 'backups/node', backup_id_3, 'database', file_page_2)
|
||||
with open(file_page1, "rb+", 0) as f:
|
||||
f.seek(84)
|
||||
f.write(b"blah")
|
||||
f.flush()
|
||||
f.close
|
||||
|
||||
file_page4 = os.path.join(backup_dir, 'backups/node', backup_id_6, 'database', file_page_5)
|
||||
file_page4 = os.path.join(
|
||||
backup_dir, 'backups/node', backup_id_6, 'database', file_page_5)
|
||||
with open(file_page4, "rb+", 0) as f:
|
||||
f.seek(42)
|
||||
f.write(b"blah")
|
||||
@ -336,51 +357,97 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
|
||||
# Validate PAGE3
|
||||
try:
|
||||
self.validate_pb(backup_dir, 'node', backup_id=backup_id_4,
|
||||
self.validate_pb(
|
||||
backup_dir, 'node',
|
||||
backup_id=backup_id_4,
|
||||
options=['--log-level-file=verbose'])
|
||||
self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of data files corruption.\n"
|
||||
" Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertTrue(
|
||||
'INFO: Validating parents for backup {0}'.format(backup_id_4) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating parents for backup {0}'.format(
|
||||
backup_id_4) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Validating backup {0}'.format(backup_id_1) in e.message
|
||||
and 'INFO: Backup {0} data files are valid'.format(backup_id_1) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating backup {0}'.format(
|
||||
backup_id_1) in e.message and
|
||||
'INFO: Backup {0} data files are valid'.format(
|
||||
backup_id_1) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Validating backup {0}'.format(backup_id_2) in e.message
|
||||
and 'INFO: Backup {0} data files are valid'.format(backup_id_2) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating backup {0}'.format(
|
||||
backup_id_2) in e.message and
|
||||
'INFO: Backup {0} data files are valid'.format(
|
||||
backup_id_2) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Validating backup {0}'.format(backup_id_3) in e.message
|
||||
and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_page1) in e.message
|
||||
and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating backup {0}'.format(
|
||||
backup_id_3) in e.message and
|
||||
'WARNING: Invalid CRC of backup file "{0}"'.format(
|
||||
file_page1) in e.message and
|
||||
'WARNING: Backup {0} data files are corrupted'.format(
|
||||
backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_4, backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'WARNING: Backup {0} is orphaned because '
|
||||
'his parent {1} is corrupted'.format(
|
||||
backup_id_4, backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_5, backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'WARNING: Backup {0} is orphaned because '
|
||||
'his parent {1} is corrupted'.format(
|
||||
backup_id_5, backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_6, backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'WARNING: Backup {0} is orphaned because '
|
||||
'his parent {1} is corrupted'.format(
|
||||
backup_id_6, backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_7, backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'WARNING: Backup {0} is orphaned because '
|
||||
'his parent {1} is corrupted'.format(
|
||||
backup_id_7, backup_id_3) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'\n Unexpected Error Message: {0}\n '
|
||||
'CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
|
||||
self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"')
|
||||
self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"')
|
||||
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"')
|
||||
self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"')
|
||||
self.assertEqual(
|
||||
'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'],
|
||||
'Backup STATUS should be "OK"')
|
||||
self.assertEqual(
|
||||
'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'],
|
||||
'Backup STATUS should be "OK"')
|
||||
self.assertEqual(
|
||||
'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'],
|
||||
'Backup STATUS should be "CORRUPT"')
|
||||
self.assertEqual(
|
||||
'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'],
|
||||
'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual(
|
||||
'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'],
|
||||
'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual(
|
||||
'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'],
|
||||
'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual(
|
||||
'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'],
|
||||
'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual(
|
||||
'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'],
|
||||
'Backup STATUS should be "OK"')
|
||||
|
||||
# Clean after yourself
|
||||
# self.del_test_dir(module_name, fname)
|
||||
@ -521,7 +588,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
corrupt file in PAGE1 backup and run validate on instance,
|
||||
expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN"""
|
||||
fname = self.id().split('.')[3]
|
||||
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
node = self.make_simple_node(
|
||||
base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
initdb_params=['--data-checksums'],
|
||||
pg_options={'wal_level': 'replica'}
|
||||
)
|
||||
@ -533,31 +601,43 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i")
|
||||
"create table t_heap as select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(0,10000) i")
|
||||
# FULL1
|
||||
backup_id_1 = self.backup_node(backup_dir, 'node', node)
|
||||
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"create table t_heap1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i")
|
||||
"create table t_heap1 as select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(0,10000) i")
|
||||
file_path_t_heap1 = node.safe_psql(
|
||||
"postgres",
|
||||
"select pg_relation_filepath('t_heap1')").rstrip()
|
||||
# PAGE1
|
||||
backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
backup_id_2 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i")
|
||||
"insert into t_heap select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(20000,30000) i")
|
||||
# PAGE2
|
||||
backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
backup_id_3 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
# FULL1
|
||||
backup_id_4 = self.backup_node(backup_dir, 'node', node)
|
||||
backup_id_4 = self.backup_node(
|
||||
backup_dir, 'node', node)
|
||||
# PAGE3
|
||||
backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page')
|
||||
backup_id_5 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
# Corrupt some file in FULL backup
|
||||
file_full = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path_t_heap1)
|
||||
file_full = os.path.join(
|
||||
backup_dir, 'backups/node', backup_id_2,
|
||||
'database', file_path_t_heap1)
|
||||
with open(file_full, "rb+", 0) as f:
|
||||
f.seek(84)
|
||||
f.write(b"blah")
|
||||
@ -566,48 +646,86 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
|
||||
# Validate Instance
|
||||
try:
|
||||
self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose'])
|
||||
self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
self.validate_pb(
|
||||
backup_dir, 'node', options=['--log-level-file=verbose'])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of data files corruption.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertTrue(
|
||||
"INFO: Validate backups of the instance 'node'" in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
"\n Unexpected Error Message: {0}\n "
|
||||
"CMD: {1}".format(repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Validating backup {0}'.format(backup_id_5) in e.message
|
||||
and 'INFO: Backup {0} data files are valid'.format(backup_id_5) in e.message
|
||||
and 'INFO: Backup {0} WAL segments are valid'.format(backup_id_5) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating backup {0}'.format(
|
||||
backup_id_5) in e.message and
|
||||
'INFO: Backup {0} data files are valid'.format(
|
||||
backup_id_5) in e.message and
|
||||
'INFO: Backup {0} WAL segments are valid'.format(
|
||||
backup_id_5) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Validating backup {0}'.format(backup_id_4) in e.message
|
||||
and 'INFO: Backup {0} data files are valid'.format(backup_id_4) in e.message
|
||||
and 'INFO: Backup {0} WAL segments are valid'.format(backup_id_4) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating backup {0}'.format(
|
||||
backup_id_4) in e.message and
|
||||
'INFO: Backup {0} data files are valid'.format(
|
||||
backup_id_4) in e.message and
|
||||
'INFO: Backup {0} WAL segments are valid'.format(
|
||||
backup_id_4) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Validating backup {0}'.format(backup_id_3) in e.message
|
||||
and 'INFO: Backup {0} data files are valid'.format(backup_id_3) in e.message
|
||||
and 'INFO: Backup {0} WAL segments are valid'.format(backup_id_3) in e.message
|
||||
and 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_3, backup_id_2) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating backup {0}'.format(
|
||||
backup_id_3) in e.message and
|
||||
'INFO: Backup {0} data files are valid'.format(
|
||||
backup_id_3) in e.message and
|
||||
'INFO: Backup {0} WAL segments are valid'.format(
|
||||
backup_id_3) in e.message and
|
||||
'WARNING: Backup {0} is orphaned because '
|
||||
'his parent {1} is corrupted'.format(
|
||||
backup_id_3, backup_id_2) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Validating backup {0}'.format(backup_id_2) in e.message
|
||||
and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message
|
||||
and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_2) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating backup {0}'.format(
|
||||
backup_id_2) in e.message and
|
||||
'WARNING: Invalid CRC of backup file "{0}"'.format(
|
||||
file_full) in e.message and
|
||||
'WARNING: Backup {0} data files are corrupted'.format(
|
||||
backup_id_2) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Validating backup {0}'.format(backup_id_1) in e.message
|
||||
and 'INFO: Backup {0} data files are valid'.format(backup_id_1) in e.message
|
||||
and 'INFO: Backup {0} WAL segments are valid'.format(backup_id_1) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'INFO: Validating backup {0}'.format(
|
||||
backup_id_1) in e.message and
|
||||
'INFO: Backup {0} data files are valid'.format(
|
||||
backup_id_1) in e.message and
|
||||
'INFO: Backup {0} WAL segments are valid'.format(
|
||||
backup_id_1) in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'INFO: Some backups are not valid' in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"')
|
||||
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"')
|
||||
self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"')
|
||||
self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"')
|
||||
self.assertEqual(
|
||||
'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'],
|
||||
'Backup STATUS should be "OK"')
|
||||
self.assertEqual(
|
||||
'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'],
|
||||
'Backup STATUS should be "CORRUPT"')
|
||||
self.assertEqual(
|
||||
'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'],
|
||||
'Backup STATUS should be "ORPHAN"')
|
||||
self.assertEqual(
|
||||
'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'],
|
||||
'Backup STATUS should be "OK"')
|
||||
self.assertEqual(
|
||||
'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'],
|
||||
'Backup STATUS should be "OK"')
|
||||
|
||||
# Clean after yourself
|
||||
# self.del_test_dir(module_name, fname)
|
||||
@ -692,7 +810,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
def test_validate_corrupt_wal_1(self):
|
||||
"""make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors"""
|
||||
fname = self.id().split('.')[3]
|
||||
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
node = self.make_simple_node(
|
||||
base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
initdb_params=['--data-checksums'],
|
||||
pg_options={'wal_level': 'replica'}
|
||||
)
|
||||
@ -724,14 +843,28 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
# Simple validate
|
||||
try:
|
||||
self.validate_pb(backup_dir, 'node')
|
||||
self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of wal segments corruption.\n"
|
||||
" Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertTrue('Possible WAL CORRUPTION' in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
'WARNING: Backup' in e.message and
|
||||
'WAL segments are corrupted' in e.message and
|
||||
"WARNING: There are not enough WAL "
|
||||
"records to consistenly restore backup" in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"')
|
||||
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"')
|
||||
self.assertEqual(
|
||||
'CORRUPT',
|
||||
self.show_pb(backup_dir, 'node', backup_id_1)['status'],
|
||||
'Backup STATUS should be "CORRUPT"')
|
||||
self.assertEqual(
|
||||
'CORRUPT',
|
||||
self.show_pb(backup_dir, 'node', backup_id_2)['status'],
|
||||
'Backup STATUS should be "CORRUPT"')
|
||||
|
||||
# Clean after yourself
|
||||
self.del_test_dir(module_name, fname)
|
||||
@ -836,30 +969,155 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
|
||||
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
|
||||
file = os.path.join(backup_dir, 'wal', 'node', wals[1])
|
||||
os.remove(file)
|
||||
|
||||
# cut out '.gz'
|
||||
if self.archive_compress:
|
||||
file = file[:-3]
|
||||
|
||||
try:
|
||||
self.validate_pb(backup_dir, 'node')
|
||||
self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of wal segment disappearance.\n"
|
||||
" Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn('WARNING: WAL segment "{0}" is absent'.format(file), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
self.assertIn('WARNING: There are not enough WAL records to consistenly restore backup {0}'.format(backup_id), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
self.assertIn('WARNING: Backup {0} WAL segments are corrupted'.format(backup_id), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
self.assertIn('INFO: Some backups are not valid', e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
self.assertTrue(
|
||||
"WARNING: WAL segment \"{0}\" is absent".format(
|
||||
file) in e.message and
|
||||
"WARNING: There are not enough WAL records to consistenly "
|
||||
"restore backup {0}".format(backup_id) in e.message and
|
||||
"WARNING: Backup {0} WAL segments are corrupted".format(
|
||||
backup_id) in e.message and
|
||||
"INFO: Some backups are not valid" in e.message,
|
||||
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"')
|
||||
self.assertEqual(
|
||||
'CORRUPT',
|
||||
self.show_pb(backup_dir, 'node', backup_id)['status'],
|
||||
'Backup {0} should have STATUS "CORRUPT"')
|
||||
|
||||
# Be paranoid and run validate again
|
||||
try:
|
||||
self.validate_pb(backup_dir, 'node')
|
||||
self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of backup corruption.\n"
|
||||
" Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
|
||||
self.assertIn(
|
||||
'INFO: Backup {0} has status CORRUPT. '
|
||||
'Skip validation.\n'.format(backup_id), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
# Clean after yourself
|
||||
self.del_test_dir(module_name, fname)
|
||||
|
||||
# @unittest.skip("skip")
|
||||
def test_validate_corrupt_wal_between_backups(self):
|
||||
"""make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors"""
|
||||
fname = self.id().split('.')[3]
|
||||
node = self.make_simple_node(
|
||||
base_dir="{0}/{1}/node".format(module_name, fname),
|
||||
initdb_params=['--data-checksums'],
|
||||
pg_options={'wal_level': 'replica'}
|
||||
)
|
||||
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
||||
self.init_pb(backup_dir)
|
||||
self.add_instance(backup_dir, 'node', node)
|
||||
self.set_archiving(backup_dir, 'node', node)
|
||||
node.start()
|
||||
|
||||
backup_id = self.backup_node(backup_dir, 'node', node)
|
||||
|
||||
# make some wals
|
||||
node.pgbench_init(scale=2)
|
||||
pgbench = node.pgbench(
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
options=["-c", "4", "-T", "10"]
|
||||
)
|
||||
pgbench.wait()
|
||||
pgbench.stdout.close()
|
||||
|
||||
with node.connect("postgres") as con:
|
||||
con.execute("CREATE TABLE tbl0005 (a text)")
|
||||
con.commit()
|
||||
|
||||
with node.connect("postgres") as con:
|
||||
res = con.execute(
|
||||
"INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
|
||||
con.commit()
|
||||
target_xid = res[0][0]
|
||||
|
||||
if self.get_version(node) < self.version_to_num('10.0'):
|
||||
walfile = node.safe_psql(
|
||||
'postgres',
|
||||
'select pg_xlogfile_name(pg_current_xlog_location())').rstrip()
|
||||
else:
|
||||
walfile = node.safe_psql(
|
||||
'postgres',
|
||||
'select pg_walfile_name(pg_current_wal_location())').rstrip()
|
||||
|
||||
if self.archive_compress:
|
||||
walfile = walfile + '.gz'
|
||||
self.switch_wal_segment(node)
|
||||
|
||||
# generate some wals
|
||||
node.pgbench_init(scale=2)
|
||||
pgbench = node.pgbench(
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
options=["-c", "4", "-T", "10"]
|
||||
)
|
||||
pgbench.wait()
|
||||
pgbench.stdout.close()
|
||||
|
||||
self.backup_node(backup_dir, 'node', node)
|
||||
|
||||
# Corrupt WAL
|
||||
wals_dir = os.path.join(backup_dir, 'wal', 'node')
|
||||
with open(os.path.join(wals_dir, walfile), "rb+", 0) as f:
|
||||
f.seek(9000)
|
||||
f.write(b"b")
|
||||
f.flush()
|
||||
f.close
|
||||
|
||||
# Validate to xid
|
||||
try:
|
||||
self.validate_pb(
|
||||
backup_dir,
|
||||
'node',
|
||||
backup_id,
|
||||
options=[
|
||||
"--log-level-console=verbose",
|
||||
"--xid={0}".format(target_xid)])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of wal segments corruption.\n"
|
||||
" Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertTrue(
|
||||
'ERROR: not enough WAL records to xid' in e.message and
|
||||
'WARNING: recovery can be done up to time' in e.message and
|
||||
"ERROR: not enough WAL records to xid {0}\n".format(
|
||||
target_xid),
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
self.assertEqual(
|
||||
'OK',
|
||||
self.show_pb(backup_dir, 'node')[0]['Status'],
|
||||
'Backup STATUS should be "OK"')
|
||||
|
||||
self.assertEqual(
|
||||
'OK',
|
||||
self.show_pb(backup_dir, 'node')[1]['Status'],
|
||||
'Backup STATUS should be "OK"')
|
||||
|
||||
# Clean after yourself
|
||||
self.del_test_dir(module_name, fname)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user