db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-07-04 05:40:26 +02:00
Code Issues Releases Activity

tests fix

Browse Source
This commit is contained in:
Grigory Smolkin
2017-06-20 13:57:23 +03:00
parent 5e034a4acc
commit 1a849cad7e
23 changed files with 1029 additions and 754 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
tests/validate_test.py
View File

@ -13,6 +13,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ValidateTest, self).__init__(*args, **kwargs)
self.module_name = 'validate'
@classmethod
def tearDownClass(cls):
@ -23,20 +24,22 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
def test_validate_wal_unreal_values(self):
"""make node with archiving, make archive backup, validate to both real and unreal values"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
with node.connect("postgres") as con:
con.execute("CREATE TABLE tbl0005 (a text)")
con.commit()
backup_id = self.backup_pb(node)
backup_id = self.backup_node(backup_dir, 'node', node)
node.pgbench_init(scale=2)
pgbench = node.pgbench(
@ -48,17 +51,18 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
pgbench.wait()
pgbench.stdout.close()
target_time = self.show_pb(node, id=backup_id)['recovery-time']
target_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time']
after_backup_time = datetime.now().replace(second=0, microsecond=0)
# Validate to real time
self.assertIn(six.b("INFO: backup validation completed successfully on"),
self.validate_pb(node, options=["--time='{0}'".format(target_time)]))
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(target_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
# Validate to unreal time
unreal_time_1 = after_backup_time - timedelta(days=2)
try:
self.validate_pb(node, options=["--time='{0}'".format(unreal_time_1)])
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_1)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
@ -68,7 +72,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
# Validate to unreal time #2
unreal_time_2 = after_backup_time + timedelta(days=2)
try:
self.validate_pb(node, options=["--time='{0}'".format(unreal_time_2)])
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_2)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
@ -83,13 +87,14 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
target_xid = res[0][0]
node.execute("postgres", "SELECT pg_switch_xlog()")
self.assertIn(six.b("INFO: backup validation completed successfully on"),
self.validate_pb(node, options=["--xid={0}".format(target_xid)]))
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
# Validate to unreal xid
unreal_xid = int(target_xid) + 1000
try:
self.validate_pb(node, options=["--xid={0}".format(unreal_xid)])
self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
@ -97,35 +102,32 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# Validate with backup ID
self.assertIn(six.b("INFO: backup validation completed successfully on"),
self.validate_pb(node, backup_id))
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.validate_pb(backup_dir, 'node', backup_id),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
# @unittest.skip("skip")
def test_validate_corrupt_wal_1(self):
"""
make node with archiving
make archive backup
corrupt all wal files
run validate, expecting error because of wal corruption
make sure that backup status is 'CORRUPT'
"""
"""make archive node, make archive backup, corrupt all wal files, run validate, expect errors"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
with node.connect("postgres") as con:
con.execute("CREATE TABLE tbl0005 (a text)")
con.commit()
backup_id = self.backup_pb(node)
backup_id = self.backup_node(backup_dir, 'node', node)
# Corrupt WAL
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, 'wal', 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals.sort()
for wal in wals:
@ -136,48 +138,43 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
# Simple validate
try:
self.validate_pb(node)
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertTrue('Possible WAL CORRUPTION' in e.message),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)
self.assertEqual('CORRUPT', self.show_pb(node, id=backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
node.stop()
# @unittest.skip("skip")
def test_validate_corrupt_wal_2(self):
"""
make node with archiving
make archive backup
corrupt all wal files
run validate to real xid, expecting error because of wal corruption
make sure that backup status is 'CORRUPT'
"""
"""make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
with node.connect("postgres") as con:
con.execute("CREATE TABLE tbl0005 (a text)")
con.commit()
backup_id = self.backup_pb(node)
backup_id = self.backup_node(backup_dir, 'node', node)
target_xid = None
with node.connect("postgres") as con:
res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
con.commit()
target_xid = res[0][0]
node.execute("postgres", "SELECT pg_switch_xlog()")
# Corrupt WAL
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, 'wal', 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals.sort()
for wal in wals:
@ -188,34 +185,34 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
# Validate to xid
try:
self.validate_pb(node, backup_id, options=['--xid={0}'.format(target_xid)])
self.validate_pb(backup_dir, 'node', backup_id, options=['--xid={0}'.format(target_xid)])
self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertTrue('Possible WAL CORRUPTION' in e.message),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)
self.assertEqual('CORRUPT', self.show_pb(node, id=backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
node.stop()
# @unittest.skip("skip")
def test_validate_wal_lost_segment_1(self):
"""
make node with archiving
make archive backup
"""make archive node, make archive full backup,
delete from archive wal segment which belong to previous backup
run validate, expecting error because of missing wal segment
make sure that backup status is 'CORRUPT'
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
pgbench = node.pgbench(
stdout=subprocess.PIPE,
@ -224,31 +221,30 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
)
pgbench.wait()
pgbench.stdout.close()
backup_id = self.backup_pb(node, backup_type='full')
backup_id = self.backup_node(backup_dir, 'node', node)
# Delete wal segment
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, 'wal', 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
file = os.path.join(self.backup_dir(node), "wal", wals[1])
file = os.path.join(backup_dir, 'wal', 'node', wals[1])
os.remove(file)
try:
self.validate_pb(node)
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertTrue(re.match('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore from [0-9a-fA-F\/]+ to [0-9a-fA-F\/]+\n\Z'.format(
file), e.message),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertIn('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore'.format(
file), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual('CORRUPT', self.show_pb(node, id=backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"')
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"')
# Be paranoid and run validate again
try:
self.validate_pb(node)
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertTrue(re.match('ERROR: Backup {0} has status: CORRUPT\n\Z'.format(backup_id), e.message),
self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
node.stop()
@ -262,15 +258,17 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
make sure that backup status is 'ERROR'
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
self.backup_pb(node, backup_type='full')
self.backup_node(backup_dir, 'node', node)
# make some wals
node.pgbench_init(scale=2)
@ -283,20 +281,22 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
pgbench.stdout.close()
# delete last wal segment
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, 'wal', 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals = map(int, wals)
file = os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))
file = os.path.join(wals_dir, '0000000' + str(max(wals)))
os.remove(file)
try:
backup_id = self.backup_pb(node, backup_type='page')
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format(
self.output, self.cmd))
except ProbackupException, e:
self.assertTrue(re.match('INFO: wait for LSN [0-9a-fA-F\/]+ in archived WAL segment .*\nWARNING: could not read WAL record at [0-9a-fA-F\/]+\nERROR: WAL segment "{0}" is absent\n\Z'.format(
file), e.message),
self.assertTrue('INFO: wait for LSN'
and 'in archived WAL segment'
and 'WARNING: could not read WAL record at'
and 'ERROR: WAL segment "{0}" is absent\n'.format(file) in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual('ERROR', self.show_pb(node)[1]['Status'], 'Backup {0} should have STATUS "ERROR"')
self.assertEqual('ERROR', self.show_pb(backup_dir, 'node')[1]['Status'], 'Backup {0} should have STATUS "ERROR"')
node.stop()