mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2025-01-08 13:44:32 +02:00
tests: more tests for partial restore
This commit is contained in:
Grigory Smolkin
parent
06805ea409
commit
e58cf5b776
448
tests/restore.py
448
tests/restore.py
@ -2429,8 +2429,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
|
||||
self.restore_node(
|
||||
backup_dir, 'node',
|
||||
node_restored_1, options=[
|
||||
"--db-include='db1'",
|
||||
"--db-exclude='db2'"])
|
||||
"--db-include=db1",
|
||||
"--db-exclude=db2"])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of 'db-exclude' and 'db-include'.\n "
|
||||
@ -2670,8 +2670,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
|
||||
self.restore_node(
|
||||
backup_dir, 'node',
|
||||
node_restored_1, options=[
|
||||
"--db-include='db1'",
|
||||
"--db-exclude='db2'"])
|
||||
"--db-include=db1",
|
||||
"--db-exclude=db2"])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because of 'db-exclude' and 'db-include'.\n "
|
||||
@ -2753,10 +2753,436 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
|
||||
# Clean after yourself
|
||||
self.del_test_dir(module_name, fname)
|
||||
|
||||
# partial restore
|
||||
# 0. basic test for db-exclude and db-include +
|
||||
# 1. old backup without support of partial restore
|
||||
# 2. FULL backup do not support partial restore, but incremental do
|
||||
# 3. database_map is missing for legal reasons, e.g. no permissions for pg_database
|
||||
# 4. database_map is empty for illegal reason
|
||||
# 5. database_map contain garbage
|
||||
def test_partial_restore_backward_compatibility_1(self):
|
||||
"""
|
||||
"""
|
||||
fname = self.id().split('.')[3]
|
||||
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
||||
node = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node'),
|
||||
set_replication=True,
|
||||
initdb_params=['--data-checksums'])
|
||||
|
||||
self.init_pb(backup_dir, old_binary=True)
|
||||
self.add_instance(backup_dir, 'node', node, old_binary=True)
|
||||
|
||||
node.slow_start()
|
||||
|
||||
# create databases
|
||||
for i in range(1, 10, 1):
|
||||
node.safe_psql(
|
||||
'postgres',
|
||||
'CREATE database db{0}'.format(i))
|
||||
|
||||
# FULL backup with old binary, without partial restore support
|
||||
backup_id = self.backup_node(
|
||||
backup_dir, 'node', node,
|
||||
old_binary=True, options=['--stream'])
|
||||
|
||||
pgdata = self.pgdata_content(node.data_dir)
|
||||
|
||||
node_restored = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node_restored'))
|
||||
node_restored.cleanup()
|
||||
|
||||
try:
|
||||
self.restore_node(
|
||||
backup_dir, 'node',
|
||||
node_restored, options=[
|
||||
"--db-exclude=db5"])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because backup do not support partial restore.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
self.output, self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn(
|
||||
"ERROR: Backup {0} has missing database_map".format(backup_id),
|
||||
e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
self.restore_node(backup_dir, 'node', node_restored)
|
||||
|
||||
pgdata_restored = self.pgdata_content(node_restored.data_dir)
|
||||
self.compare_pgdata(pgdata, pgdata_restored)
|
||||
|
||||
# incremental backup with partial restore support
|
||||
for i in range(11, 15, 1):
|
||||
node.safe_psql(
|
||||
'postgres',
|
||||
'CREATE database db{0}'.format(i))
|
||||
|
||||
# get db list
|
||||
db_list_raw = node.safe_psql(
|
||||
'postgres',
|
||||
'SELECT to_json(a) '
|
||||
'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
|
||||
db_list_splitted = db_list_raw.splitlines()
|
||||
db_list = {}
|
||||
for line in db_list_splitted:
|
||||
line = json.loads(line)
|
||||
db_list[line['datname']] = line['oid']
|
||||
|
||||
backup_id = self.backup_node(
|
||||
backup_dir, 'node', node,
|
||||
backup_type='delta', options=['--stream'])
|
||||
|
||||
# get etalon
|
||||
node_restored.cleanup()
|
||||
self.restore_node(backup_dir, 'node', node_restored)
|
||||
self.truncate_every_file_in_dir(
|
||||
os.path.join(
|
||||
node_restored.data_dir, 'base', db_list['db5']))
|
||||
self.truncate_every_file_in_dir(
|
||||
os.path.join(
|
||||
node_restored.data_dir, 'base', db_list['db14']))
|
||||
pgdata_restored = self.pgdata_content(node_restored.data_dir)
|
||||
|
||||
# get new node
|
||||
node_restored_1 = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node_restored_1'))
|
||||
node_restored_1.cleanup()
|
||||
|
||||
self.restore_node(
|
||||
backup_dir, 'node',
|
||||
node_restored_1, options=[
|
||||
"--db-exclude=db5",
|
||||
"--db-exclude=db14"])
|
||||
pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir)
|
||||
|
||||
self.compare_pgdata(pgdata_restored, pgdata_restored_1)
|
||||
|
||||
def test_partial_restore_backward_compatibility_merge(self):
|
||||
"""
|
||||
"""
|
||||
fname = self.id().split('.')[3]
|
||||
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
||||
node = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node'),
|
||||
set_replication=True,
|
||||
initdb_params=['--data-checksums'])
|
||||
|
||||
self.init_pb(backup_dir, old_binary=True)
|
||||
self.add_instance(backup_dir, 'node', node, old_binary=True)
|
||||
|
||||
node.slow_start()
|
||||
|
||||
# create databases
|
||||
for i in range(1, 10, 1):
|
||||
node.safe_psql(
|
||||
'postgres',
|
||||
'CREATE database db{0}'.format(i))
|
||||
|
||||
# FULL backup with old binary, without partial restore support
|
||||
backup_id = self.backup_node(
|
||||
backup_dir, 'node', node,
|
||||
old_binary=True, options=['--stream'])
|
||||
|
||||
pgdata = self.pgdata_content(node.data_dir)
|
||||
|
||||
node_restored = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node_restored'))
|
||||
node_restored.cleanup()
|
||||
|
||||
try:
|
||||
self.restore_node(
|
||||
backup_dir, 'node',
|
||||
node_restored, options=[
|
||||
"--db-exclude=db5"])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because backup do not support partial restore.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
self.output, self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn(
|
||||
"ERROR: Backup {0} has missing database_map".format(backup_id),
|
||||
e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
self.restore_node(backup_dir, 'node', node_restored)
|
||||
|
||||
pgdata_restored = self.pgdata_content(node_restored.data_dir)
|
||||
self.compare_pgdata(pgdata, pgdata_restored)
|
||||
|
||||
# incremental backup with partial restore support
|
||||
for i in range(11, 15, 1):
|
||||
node.safe_psql(
|
||||
'postgres',
|
||||
'CREATE database db{0}'.format(i))
|
||||
|
||||
# get db list
|
||||
db_list_raw = node.safe_psql(
|
||||
'postgres',
|
||||
'SELECT to_json(a) '
|
||||
'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
|
||||
db_list_splitted = db_list_raw.splitlines()
|
||||
db_list = {}
|
||||
for line in db_list_splitted:
|
||||
line = json.loads(line)
|
||||
db_list[line['datname']] = line['oid']
|
||||
|
||||
backup_id = self.backup_node(
|
||||
backup_dir, 'node', node,
|
||||
backup_type='delta', options=['--stream'])
|
||||
|
||||
# get etalon
|
||||
node_restored.cleanup()
|
||||
self.restore_node(backup_dir, 'node', node_restored)
|
||||
self.truncate_every_file_in_dir(
|
||||
os.path.join(
|
||||
node_restored.data_dir, 'base', db_list['db5']))
|
||||
self.truncate_every_file_in_dir(
|
||||
os.path.join(
|
||||
node_restored.data_dir, 'base', db_list['db14']))
|
||||
pgdata_restored = self.pgdata_content(node_restored.data_dir)
|
||||
|
||||
# get new node
|
||||
node_restored_1 = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node_restored_1'))
|
||||
node_restored_1.cleanup()
|
||||
|
||||
# merge
|
||||
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
|
||||
|
||||
self.restore_node(
|
||||
backup_dir, 'node',
|
||||
node_restored_1, options=[
|
||||
"--db-exclude=db5",
|
||||
"--db-exclude=db14"])
|
||||
pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir)
|
||||
|
||||
self.compare_pgdata(pgdata_restored, pgdata_restored_1)
|
||||
|
||||
def test_missing_database_map(self):
|
||||
"""
|
||||
"""
|
||||
fname = self.id().split('.')[3]
|
||||
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
||||
node = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node'),
|
||||
set_replication=True,
|
||||
initdb_params=['--data-checksums'])
|
||||
|
||||
self.init_pb(backup_dir)
|
||||
self.add_instance(backup_dir, 'node', node)
|
||||
|
||||
node.slow_start()
|
||||
|
||||
# create databases
|
||||
for i in range(1, 10, 1):
|
||||
node.safe_psql(
|
||||
'postgres',
|
||||
'CREATE database db{0}'.format(i))
|
||||
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"CREATE DATABASE backupdb")
|
||||
|
||||
if self.get_version(node) > self.version_to_num('10.0'):
|
||||
# bootstrap for 10/11
|
||||
node.safe_psql(
|
||||
"backupdb",
|
||||
"REVOKE ALL on SCHEMA public from public; "
|
||||
"REVOKE ALL on SCHEMA pg_catalog from public; "
|
||||
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM public; "
|
||||
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public; "
|
||||
"CREATE ROLE backup WITH LOGIN REPLICATION; "
|
||||
"GRANT CONNECT ON DATABASE postgres to backup; "
|
||||
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
|
||||
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
|
||||
# we use it for partial restore and checkdb
|
||||
# "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; "
|
||||
# "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, oid) TO backup;"
|
||||
)
|
||||
else:
|
||||
# bootstrap for 9.5/9.6
|
||||
node.safe_psql(
|
||||
"backupdb",
|
||||
"REVOKE ALL on SCHEMA public from public; "
|
||||
"REVOKE ALL on SCHEMA pg_catalog from public; "
|
||||
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM public; "
|
||||
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public; "
|
||||
"CREATE ROLE backup WITH LOGIN REPLICATION; "
|
||||
"GRANT CONNECT ON DATABASE postgres to backup; "
|
||||
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
|
||||
# we use it for ptrack
|
||||
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
|
||||
# we use it for partial restore and checkdb
|
||||
# "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; "
|
||||
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; "
|
||||
# "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, oid) TO backup;"
|
||||
)
|
||||
|
||||
# FULL backup without database_map
|
||||
backup_id = self.backup_node(
|
||||
backup_dir, 'node', node, datname='backupdb',
|
||||
options=['--stream', "-U", "backup"])
|
||||
|
||||
pgdata = self.pgdata_content(node.data_dir)
|
||||
|
||||
node_restored = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node_restored'))
|
||||
node_restored.cleanup()
|
||||
|
||||
# backup has missing database_map and that is legal
|
||||
try:
|
||||
self.restore_node(
|
||||
backup_dir, 'node', node_restored,
|
||||
options=["--db-exclude=db5", "--db-exclude=db9"])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because user do not have pg_database access.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
self.output, self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn(
|
||||
"ERROR: Backup {0} has missing database_map, "
|
||||
"partial restore is impossible.".format(
|
||||
backup_id), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
try:
|
||||
self.restore_node(
|
||||
backup_dir, 'node', node_restored,
|
||||
options=["--db-include=db1"])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because user do not have pg_database access.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
self.output, self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn(
|
||||
"ERROR: Backup {0} has missing database_map, "
|
||||
"partial restore is impossible.".format(
|
||||
backup_id), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
# check that simple restore is still possible
|
||||
self.restore_node(backup_dir, 'node', node_restored)
|
||||
|
||||
pgdata_restored = self.pgdata_content(node_restored.data_dir)
|
||||
self.compare_pgdata(pgdata, pgdata_restored)
|
||||
|
||||
def test_empty_and_mangled_database_map(self):
|
||||
"""
|
||||
"""
|
||||
fname = self.id().split('.')[3]
|
||||
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
||||
node = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node'),
|
||||
set_replication=True,
|
||||
initdb_params=['--data-checksums'])
|
||||
|
||||
self.init_pb(backup_dir)
|
||||
self.add_instance(backup_dir, 'node', node)
|
||||
|
||||
node.slow_start()
|
||||
|
||||
# create databases
|
||||
for i in range(1, 10, 1):
|
||||
node.safe_psql(
|
||||
'postgres',
|
||||
'CREATE database db{0}'.format(i))
|
||||
|
||||
# FULL backup with database_map
|
||||
backup_id = self.backup_node(
|
||||
backup_dir, 'node', node, options=['--stream'])
|
||||
pgdata = self.pgdata_content(node.data_dir)
|
||||
|
||||
# truncate database_map
|
||||
path = os.path.join(
|
||||
backup_dir, 'backups', 'node',
|
||||
backup_id, 'database', 'database_map')
|
||||
with open(path, "w") as f:
|
||||
f.close()
|
||||
|
||||
node_restored = self.make_simple_node(
|
||||
base_dir=os.path.join(module_name, fname, 'node_restored'))
|
||||
node_restored.cleanup()
|
||||
|
||||
try:
|
||||
self.restore_node(
|
||||
backup_dir, 'node', node_restored,
|
||||
options=["--db-include=db1", '--no-validate'])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because database_map is empty.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
self.output, self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn(
|
||||
"ERROR: Backup {0} has empty or mangled database_map, "
|
||||
"partial restore is impossible".format(backup_id), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
try:
|
||||
self.restore_node(
|
||||
backup_dir, 'node', node_restored,
|
||||
options=["--db-exclude=db1", '--no-validate'])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because database_map is empty.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
self.output, self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn(
|
||||
"ERROR: Backup {0} has empty or mangled database_map, "
|
||||
"partial restore is impossible".format(backup_id), e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
# mangle database_map
|
||||
with open(path, "w") as f:
|
||||
f.write("42")
|
||||
f.close()
|
||||
|
||||
try:
|
||||
self.restore_node(
|
||||
backup_dir, 'node', node_restored,
|
||||
options=["--db-include=db1", '--no-validate'])
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because database_map is empty.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
self.output, self.cmd))
|
||||
except ProbackupException as e:
|
||||
self.assertIn(
|
||||
'ERROR: field "dbOid" is not found in the line 42 of '
|
||||
'the file backup_content.control', e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
|
||||
# check that simple restore is still possible
|
||||
self.restore_node(
|
||||
backup_dir, 'node', node_restored, options=['--no-validate'])
|
||||
|
||||
pgdata_restored = self.pgdata_content(node_restored.data_dir)
|
||||
self.compare_pgdata(pgdata, pgdata_restored)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user