db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

tests: now ptrack tests by default are disabled, they can be enables via PG_PROBACKUP_PTRACK env variable. Multiple fixes, especially for PG 9.5

Browse Source
This commit is contained in:
Grigory Smolkin
2019-07-12 18:01:28 +03:00
parent 3d9bb1b076
commit b38692f464
16 changed files with 284 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Readme.md
+3
View File
@@ -29,6 +29,9 @@ Remote backup depends on key authentithication to local machine via ssh as curre
Run suit of basic simple tests:
export PG_PROBACKUP_TEST_BASIC=ON
Run ptrack tests:
export PG_PROBACKUP_PTRACK=ON
Usage:
pip install testgres
tests/__init__.py
+4 -1
View File
@@ -37,7 +37,6 @@ def load_tests(loader, tests, pattern):
suite.addTests(loader.loadTestsFromModule(merge))
suite.addTests(loader.loadTestsFromModule(option))
suite.addTests(loader.loadTestsFromModule(page))
# suite.addTests(loader.loadTestsFromModule(ptrack))
suite.addTests(loader.loadTestsFromModule(pgpro560))
suite.addTests(loader.loadTestsFromModule(pgpro589))
suite.addTests(loader.loadTestsFromModule(pgpro2068))
@@ -50,6 +49,10 @@ def load_tests(loader, tests, pattern):
suite.addTests(loader.loadTestsFromModule(time_stamp))
suite.addTests(loader.loadTestsFromModule(validate))
if 'PG_PROBACKUP_PTRACK' in os.environ:
if os.environ['PG_PROBACKUP_PTRACK'] == 'ON':
suite.addTests(loader.loadTestsFromModule(ptrack))
return suite
# test_pgpro434_2 unexpected success
tests/archive.py
+11 -12
View File
@@ -228,10 +228,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
@@ -258,18 +256,19 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
log_file = os.path.join(backup_dir, 'log/pg_probackup.log')
with open(log_file, 'r') as f:
log_content = f.read()
self.assertNotIn(
"ERROR: pg_stop_backup doesn't answer",
log_content,
"pg_stop_backup timeouted")
self.assertIn(
"ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it",
log_content)
log_file = os.path.join(node.logs_dir, 'postgresql.log')
with open(log_file, 'r') as f:
log_content = f.read()
self.assertNotIn(
'FailedAssertion',
log_content,
'PostgreSQL crashed because of a failed assert')
self.assertNotIn(
'FailedAssertion',
log_content,
'PostgreSQL crashed because of a failed assert')
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/backup.py
+113 -61
View File
@@ -15,6 +15,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# PGPRO-707
def test_backup_modes_archive(self):
"""standart backup modes with ARCHIVE WAL method"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -109,6 +112,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_incremental_backup_without_full(self):
"""page-level backup without validated full backup"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -168,8 +174,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on'})
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
@@ -230,6 +235,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_ptrack_threads(self):
"""ptrack multi thread backup mode"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -258,6 +266,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_ptrack_threads_stream(self):
"""ptrack multi thread backup mode and stream"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -287,6 +298,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_page_corruption_heal_via_ptrack_1(self):
"""make node, corrupt some page, check that backup failed"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -343,6 +357,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_page_corruption_heal_via_ptrack_2(self):
"""make node, corrupt some page, check that backup failed"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -988,6 +1005,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_drop_rel_during_backup_ptrack(self):
""""""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
@@ -1492,10 +1512,12 @@ class BackupTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
# pg_options={'ptrack_enable': 'on'},
initdb_params=['--data-checksums'],
pg_options={
'archive_timeout': '30s'})
)
if self.ptrack:
node.append_conf('postgresql.auto.conf', 'ptrack_enable = on')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -1506,58 +1528,87 @@ class BackupTest(ProbackupTest, unittest.TestCase):
'postgres',
'CREATE DATABASE backupdb')
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
# for partial restore, checkdb and ptrack
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
# for exclusive backup for PG 9.5 and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
if self.get_version(node) < 90600:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
else:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
# ptrack functions
# for fname in [
# 'oideq(oid, oid)',
# 'ptrack_version()',
# 'pg_ptrack_clear()',
# 'pg_ptrack_control_lsn()',
# 'pg_ptrack_get_and_clear_db(oid, oid)',
# 'pg_ptrack_get_and_clear(oid, oid)',
# 'pg_ptrack_get_block_2(oid, oid, oid, bigint)']:
# try:
# node.safe_psql(
# "backupdb",
# "GRANT EXECUTE ON FUNCTION pg_catalog.{0} "
# "TO backup".format(fname))
# except:
# pass
if self.ptrack:
for fname in [
'oideq(oid, oid)',
'ptrack_version()',
'pg_ptrack_clear()',
'pg_ptrack_control_lsn()',
'pg_ptrack_get_and_clear_db(oid, oid)',
'pg_ptrack_get_and_clear(oid, oid)',
'pg_ptrack_get_block_2(oid, oid, oid, bigint)']:
# try:
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.{0} "
"TO backup".format(fname))
# except:
# pass
# FULL backup
self.backup_node(
@@ -1584,12 +1635,13 @@ class BackupTest(ProbackupTest, unittest.TestCase):
datname='backupdb', options=['--stream', '-U', 'backup'])
# PTRACK
# self.backup_node(
# backup_dir, 'node', node, backup_type='ptrack',
# datname='backupdb', options=['-U', 'backup'])
# self.backup_node(
# backup_dir, 'node', node, backup_type='ptrack',
# datname='backupdb', options=['--stream', '-U', 'backup'])
if self.ptrack:
self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
datname='backupdb', options=['-U', 'backup'])
self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
datname='backupdb', options=['--stream', '-U', 'backup'])
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/checkdb.py
+1 -1
View File
@@ -229,7 +229,7 @@ class CheckdbTest(ProbackupTest, unittest.TestCase):
# create two databases
node.safe_psql("postgres", "create database db1")
try:
node.safe_psql(
node.safe_psql(
"db1",
"create extension amcheck")
except QueryException as e:
tests/compatibility.py
+11 -6
View File
@@ -217,6 +217,10 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_backward_compatibility_ptrack(self):
"""Description in jira issue PGPRO-434"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
@@ -224,8 +228,9 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'}
)
'autovacuum': 'off',
'ptrack_enable': 'on'})
self.init_pb(backup_dir, old_binary=True)
self.show_pb(backup_dir)
@@ -262,7 +267,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Delta BACKUP with old binary
# ptrack BACKUP with old binary
pgbench = node.pgbench(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
@@ -272,7 +277,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
pgbench.stdout.close()
self.backup_node(
backup_dir, 'node', node, backup_type='delta',
backup_dir, 'node', node, backup_type='ptrack',
old_binary=True)
if self.paranoia:
@@ -287,7 +292,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Delta BACKUP with new binary
# Ptrack BACKUP with new binary
pgbench = node.pgbench(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
@@ -297,7 +302,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
pgbench.stdout.close()
self.backup_node(
backup_dir, 'node', node, backup_type='delta')
backup_dir, 'node', node, backup_type='ptrack')
if self.paranoia:
pgdata = self.pgdata_content(node.data_dir)
tests/compression.py
+39 -59
View File
@@ -23,11 +23,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -59,15 +55,15 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
options=[
'--stream', '--compress-algorithm=zlib'])
# PTRACK BACKUP
# DELTA BACKUP
node.safe_psql(
"postgres",
"insert into t_heap select i as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(512,768) i")
ptrack_result = node.execute("postgres", "SELECT * FROM t_heap")
ptrack_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
delta_result = node.execute("postgres", "SELECT * FROM t_heap")
delta_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='delta',
options=['--stream', '--compress-algorithm=zlib'])
# Drop Node
@@ -105,11 +101,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(page_result, page_result_new)
node.cleanup()
# Check ptrack backup
# Check delta backup
self.assertIn(
"INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
"INFO: Restore of backup {0} completed.".format(delta_backup_id),
self.restore_node(
backup_dir, 'node', node, backup_id=ptrack_backup_id,
backup_dir, 'node', node, backup_id=delta_backup_id,
options=[
"-j", "4", "--immediate",
"--recovery-target-action=promote"]),
@@ -117,8 +113,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
node.slow_start()
ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(ptrack_result, ptrack_result_new)
delta_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(delta_result, delta_result_new)
node.cleanup()
# Clean after yourself
@@ -135,11 +131,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -167,14 +159,14 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, backup_type='page',
options=["--compress-algorithm=zlib"])
# PTRACK BACKUP
# DELTA BACKUP
node.safe_psql(
"postgres",
"insert into t_heap select i as id, md5(i::text) as text, "
"md5(i::text)::tsvector as tsvector from generate_series(0,3) i")
ptrack_result = node.execute("postgres", "SELECT * FROM t_heap")
ptrack_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
delta_result = node.execute("postgres", "SELECT * FROM t_heap")
delta_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='delta',
options=['--compress-algorithm=zlib'])
# Drop Node
@@ -212,11 +204,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(page_result, page_result_new)
node.cleanup()
# Check ptrack backup
# Check delta backup
self.assertIn(
"INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
"INFO: Restore of backup {0} completed.".format(delta_backup_id),
self.restore_node(
backup_dir, 'node', node, backup_id=ptrack_backup_id,
backup_dir, 'node', node, backup_id=delta_backup_id,
options=[
"-j", "4", "--immediate",
"--recovery-target-action=promote"]),
@@ -224,8 +216,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
node.slow_start()
ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(ptrack_result, ptrack_result_new)
delta_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(delta_result, delta_result_new)
node.cleanup()
# Clean after yourself
@@ -242,11 +234,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -275,7 +263,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, backup_type='page',
options=['--stream', '--compress-algorithm=pglz'])
# PTRACK BACKUP
# DELTA BACKUP
node.safe_psql(
"postgres",
"insert into t_heap select i as id, md5(i::text) as text, "
@@ -283,7 +271,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
"from generate_series(512,768) i")
ptrack_result = node.execute("postgres", "SELECT * FROM t_heap")
ptrack_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
backup_dir, 'node', node, backup_type='delta',
options=['--stream', '--compress-algorithm=pglz'])
# Drop Node
@@ -321,11 +309,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(page_result, page_result_new)
node.cleanup()
# Check ptrack backup
# Check delta backup
self.assertIn(
"INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
"INFO: Restore of backup {0} completed.".format(delta_backup_id),
self.restore_node(
backup_dir, 'node', node, backup_id=ptrack_backup_id,
backup_dir, 'node', node, backup_id=delta_backup_id,
options=[
"-j", "4", "--immediate",
"--recovery-target-action=promote"]),
@@ -333,8 +321,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
node.slow_start()
ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(ptrack_result, ptrack_result_new)
delta_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(delta_result, delta_result_new)
node.cleanup()
# Clean after yourself
@@ -351,11 +339,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -384,15 +368,15 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, backup_type='page',
options=['--compress-algorithm=pglz'])
# PTRACK BACKUP
# DELTA BACKUP
node.safe_psql(
"postgres",
"insert into t_heap select i as id, md5(i::text) as text, "
"md5(i::text)::tsvector as tsvector "
"from generate_series(200,300) i")
ptrack_result = node.execute("postgres", "SELECT * FROM t_heap")
ptrack_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
delta_result = node.execute("postgres", "SELECT * FROM t_heap")
delta_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='delta',
options=['--compress-algorithm=pglz'])
# Drop Node
@@ -430,11 +414,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(page_result, page_result_new)
node.cleanup()
# Check ptrack backup
# Check delta backup
self.assertIn(
"INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
"INFO: Restore of backup {0} completed.".format(delta_backup_id),
self.restore_node(
backup_dir, 'node', node, backup_id=ptrack_backup_id,
backup_dir, 'node', node, backup_id=delta_backup_id,
options=[
"-j", "4", "--immediate",
"--recovery-target-action=promote"]),
@@ -442,8 +426,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
node.slow_start()
ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(ptrack_result, ptrack_result_new)
delta_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(delta_result, delta_result_new)
node.cleanup()
# Clean after yourself
@@ -460,11 +444,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
tests/delete.py
+3
View File
@@ -188,6 +188,9 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_delete_increment_ptrack(self):
"""delete increment and all after him"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
tests/delta.py
+6
View File
@@ -1078,6 +1078,9 @@ class DeltaTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_delta_corruption_heal_via_ptrack_1(self):
"""make node, corrupt some page, check that backup failed"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -1135,6 +1138,9 @@ class DeltaTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_page_corruption_heal_via_ptrack_2(self):
"""make node, corrupt some page, check that backup failed"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
tests/exclude.py
+4 -8
View File
@@ -20,10 +20,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'max_wal_senders': '2',
'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'})
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -102,7 +99,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase):
def test_exclude_unlogged_tables_1(self):
"""
make node without archiving, create unlogged table, take full backup,
alter table to unlogged, take ptrack backup, restore ptrack backup,
alter table to unlogged, take delta backup, restore delta backup,
check that PGDATA`s are physically the same
"""
fname = self.id().split('.')[3]
@@ -113,8 +110,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase):
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off',
"shared_buffers": "10MB",
'ptrack_enable': 'on'})
"shared_buffers": "10MB"})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -138,7 +134,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase):
node.safe_psql('postgres', "alter table test set logged")
self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
backup_dir, 'node', node, backup_type='delta',
options=['--stream']
)
tests/false_positive.py
+9 -2
View File
@@ -57,8 +57,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on'})
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
@@ -114,6 +113,10 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
def test_ptrack_concurrent_get_and_clear_1(self):
"""make node, make full and ptrack stream backups,"
" restore them and check data correctness"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
@@ -195,6 +198,10 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
def test_ptrack_concurrent_get_and_clear_2(self):
"""make node, make full and ptrack stream backups,"
" restore them and check data correctness"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
tests/helpers/ptrack_helpers.py
+7 -1
View File
@@ -271,7 +271,13 @@ class ProbackupTest(object):
self.remote_user = None
if 'PGPROBACKUP_SSH_REMOTE' in self.test_env:
self.remote = True
if self.test_env['PGPROBACKUP_SSH_REMOTE'] == 'ON':
self.remote = True
self.ptrack = False
if 'PG_PROBACKUP_PTRACK' in self.test_env:
if self.test_env['PG_PROBACKUP_PTRACK'] == 'ON':
self.ptrack = True
@property
def pg_config_version(self):
tests/logging.py
+3 -4
View File
@@ -142,7 +142,7 @@ class LogTest(ProbackupTest, unittest.TestCase):
log_file_size)
self.assertNotIn(
'WARNING:',
'WARNING: cannot read creation timestamp from rotation file',
output)
self.assertTrue(os.path.isfile(rotation_file_path))
@@ -166,7 +166,6 @@ class LogTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node',
options=['--log-rotation-age=1d'])
self.backup_node(
backup_dir, 'node', node,
options=[
@@ -212,7 +211,7 @@ class LogTest(ProbackupTest, unittest.TestCase):
return_id=False)
self.assertNotIn(
'WARNING:',
'WARNING: missing rotation file:',
output)
# check that log file wasn`t rotated
@@ -291,7 +290,7 @@ class LogTest(ProbackupTest, unittest.TestCase):
return_id=False)
self.assertNotIn(
'WARNING:',
'WARNING: rotation file',
output)
# check that log file wasn`t rotated
tests/merge.py
+22 -8
View File
@@ -826,6 +826,9 @@ class MergeTest(ProbackupTest, unittest.TestCase):
take page backup, merge full and page,
restore last page backup and check data correctness
"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
@@ -1651,8 +1654,6 @@ class MergeTest(ProbackupTest, unittest.TestCase):
os.remove(file_to_remove)
# Try to continue failed MERGE
#print(backup_id)
#exit(1)
self.merge_backup(backup_dir, "node", backup_id)
self.assertEqual(
@@ -1875,15 +1876,28 @@ class MergeTest(ProbackupTest, unittest.TestCase):
pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum'])
pgbench.wait()
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgdata = self.pgdata_content(node.data_dir)
node.cleanup()
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
self.restore_node(
backup_dir, 'node',
node_restored, backup_id=backup_id)
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# check that merged backup has the same state as
node_restored.cleanup()
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
self.restore_node(backup_dir, 'node', node, backup_id=backup_id)
pgdata_restored = self.pgdata_content(node.data_dir)
self.restore_node(
backup_dir, 'node',
node_restored, backup_id=backup_id)
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
tests/restore.py
+44 -27
View File
@@ -492,6 +492,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_restore_full_ptrack_archive(self):
"""recovery to latest from archive full+ptrack backups"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -540,6 +543,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_restore_ptrack(self):
"""recovery to latest from archive full+ptrack+ptrack backups"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -595,6 +601,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_restore_full_ptrack_stream(self):
"""recovery in stream mode to latest from full + ptrack backups"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -647,6 +656,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
recovery to latest from full + ptrack backups
with loads when ptrack backup do
"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -711,6 +723,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
recovery to latest from full + page backups
with loads when full backup do
"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -2028,33 +2043,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.slow_start()
# Restore with recovery target lsn
node.cleanup()
self.restore_node(
backup_dir, 'node', node,
options=[
'--recovery-target-lsn={0}'.format(target_lsn),
"--recovery-target-action=promote",
'--recovery-target-timeline=1',
])
with open(recovery_conf, 'r') as f:
recovery_conf_content = f.read()
self.assertIn(
"recovery_target_lsn = '{0}'".format(target_lsn),
recovery_conf_content)
self.assertIn(
"recovery_target_action = 'promote'",
recovery_conf_content)
self.assertIn(
"recovery_target_timeline = '1'",
recovery_conf_content)
node.slow_start()
# Restore with recovery target name
node.cleanup()
self.restore_node(
@@ -2082,6 +2070,35 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.slow_start()
# Restore with recovery target lsn
if self.get_version(node) >= 100000:
node.cleanup()
self.restore_node(
backup_dir, 'node', node,
options=[
'--recovery-target-lsn={0}'.format(target_lsn),
"--recovery-target-action=promote",
'--recovery-target-timeline=1',
])
with open(recovery_conf, 'r') as f:
recovery_conf_content = f.read()
self.assertIn(
"recovery_target_lsn = '{0}'".format(target_lsn),
recovery_conf_content)
self.assertIn(
"recovery_target_action = 'promote'",
recovery_conf_content)
self.assertIn(
"recovery_target_timeline = '1'",
recovery_conf_content)
node.slow_start()
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/retention.py
+4
View File
@@ -1281,6 +1281,10 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'])
if self.get_version(node) < 90600:
self.del_test_dir(module_name, fname)
return unittest.skip('Skipped because ptrack support is disabled')
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)