From 2b02bd40945d89432b4891edd23bf33906a6a21a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 May 2017 13:01:30 +0300 Subject: [PATCH] pgpro589 added --- tests/__init__.py | 7 ++- tests/pgpro561.py | 3 ++ tests/pgpro589.py | 60 ++++++++++++++++++++++ tests/pgpro668.py | 110 ---------------------------------------- tests/pgpro688.py | 45 ++++++++++++++++ tests/pgpro702.py | 39 ++++++++++++++ tests/ptrack_helpers.py | 26 +++++++--- tests/restore_test.py | 60 ++++++++++++++++++++-- tests/validate_test.py | 2 + 9 files changed, 228 insertions(+), 124 deletions(-) create mode 100644 tests/pgpro589.py delete mode 100644 tests/pgpro668.py create mode 100644 tests/pgpro688.py create mode 100644 tests/pgpro702.py diff --git a/tests/__init__.py b/tests/__init__.py index 036de2e7..39c9f625 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,13 +5,16 @@ from . import init_test, option_test, show_test, \ retention_test, ptrack_clean, ptrack_cluster, \ ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ - ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro561, pgpro668 + ptrack_vacuum_full, ptrack_vacuum_truncate, common_archive_test, \ + pgpro561, pgpro688, pgpro702, pgpro589 def load_tests(loader, tests, pattern): suite = unittest.TestSuite() suite.addTests(loader.loadTestsFromModule(pgpro561)) - suite.addTests(loader.loadTestsFromModule(pgpro668)) + suite.addTests(loader.loadTestsFromModule(pgpro589)) + suite.addTests(loader.loadTestsFromModule(pgpro688)) + suite.addTests(loader.loadTestsFromModule(pgpro702)) suite.addTests(loader.loadTestsFromModule(init_test)) suite.addTests(loader.loadTestsFromModule(option_test)) suite.addTests(loader.loadTestsFromModule(show_test)) diff --git a/tests/pgpro561.py b/tests/pgpro561.py index cb4b28a4..0d94d1a4 100644 --- a/tests/pgpro561.py +++ b/tests/pgpro561.py @@ -57,3 +57,6 @@ class CommonArchiveDir(ProbackupTest, unittest.TestCase): res = node1.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) + + master.stop() + node1.stop() diff --git a/tests/pgpro589.py b/tests/pgpro589.py new file mode 100644 index 00000000..9f15643f --- /dev/null +++ b/tests/pgpro589.py @@ -0,0 +1,60 @@ +import unittest +import os +import six +from .ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class LsnCheck(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(LsnCheck, self).__init__(*args, **kwargs) + +# @classmethod +# def tearDownClass(cls): +# stop_all() +# @unittest.expectedFailure + def test_pgpro589(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check that no files where copied to backup catalogue + EXPECTED TO FAIL + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() + + self.assertEqual(self.init_pb(node), six.b("")) + proc = self.backup_pb( + node, backup_type='full', options=['--archive-timeout=1'], async=True) + + content = proc.stderr.read() + self.assertEqual(True, 'wait for LSN' in repr(content), + 'No Wait for LSN') + self.assertEqual(True, 'could not be archived' in repr(content), + 'No Fail Archiving Message') + + id = self.show_pb(node)[0]['ID'] + self.assertEqual('ERROR', self.show_pb(node, id=id)['status'], 'Backup should have ERROR status') + #print self.backup_dir(node) + file = os.path.join(self.backup_dir(node), 'backups', id, 'database', path) + self.assertEqual(False, os.path.isfile(file), + '\n Start LSN was not found in archive but datafiles where copied to backup catalogue.\n For example: {0}\n It is not optimal'.format(file)) diff --git a/tests/pgpro668.py b/tests/pgpro668.py deleted file mode 100644 index 398bac3c..00000000 --- a/tests/pgpro668.py +++ /dev/null @@ -1,110 +0,0 @@ -import unittest -import os -import six -from .ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -from testgres import stop_all -import subprocess -from sys import exit - - -class RecoveryWithTimeTarget(ProbackupTest, unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(RecoveryWithTimeTarget, self).__init__(*args, **kwargs) - -# @classmethod -# def tearDownClass(cls): -# stop_all() - - def test_archive_node_backup_stream_restore_to_recovery_time(self): - """ - make node with archiving, make stream backup, - get Recovery Time, try to make pitr to Recovery Time - """ - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname), - set_archiving=True, - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) - node.start() - - self.assertEqual(self.init_pb(node), six.b("")) - id = self.backup_pb(node, backup_type='full', options=["--stream"]) - recovery_time = self.show_pb(node, id=id)['recovery-time'] - - node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) - node.cleanup() - - self.restore_pb(node, options=['--time="{0}"'.format(recovery_time)]) - node.start({"-t": "600"}) - self.assertEqual(True, node.status()) - - def test_validate_to_recovery_time(self): - """ - make node with archiving, make stream backup, - get Recovery Time, validate to Recovery Time - EXPECT VALIDATE TO FAIL - Waiting PGPRO-688 - """ - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname), - set_archiving=True, - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) - node.start() - - self.assertEqual(self.init_pb(node), six.b("")) - id = self.backup_pb(node, backup_type='full', options=["--stream"]) - recovery_time = self.show_pb(node, id=id)['recovery-time'] - - # Optional - #node.psql("postgres", "select pg_create_restore_point('123')") - #node.psql("postgres", "select txid_current()") - #node.psql("postgres", "select pg_switch_xlog()") - self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, options=["--time='{0}'".format(recovery_time)])) - #### - - node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) - node.cleanup() - - self.restore_pb(node, options=['--time="{0}"'.format(recovery_time)]) - node.start({"-t": "600"}) - self.assertEqual(True, node.status()) - - def test_archive_node_backup_stream_additional_commit_pitr(self): - """ - make node with archiving, make stream backup, create table t_heap, - try to make pitr to Recovery Time, check that t_heap do not exists - """ - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname), - set_archiving=True, - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) - node.start() - - self.assertEqual(self.init_pb(node), six.b("")) - id = self.backup_pb(node, backup_type='full', options=["--stream"]) - node.psql("postgres", "create table t_heap(a int)") - node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) - node.cleanup() - recovery_time = self.show_pb(node, id=id)['recovery-time'] - self.restore_pb(node, - options=["-j", "4", '--time="{0}"'.format(recovery_time)] - ) - node.start({"-t": "600"}) - res = node.psql("postgres", 'select * from t_heap') - self.assertEqual(True, 'does not exist' in res[2]) - - -# Need test for validate time with autonomous backup without archiving -# We need to forbid validation of autonomous backup by time or xid -# if archiving is not set diff --git a/tests/pgpro688.py b/tests/pgpro688.py new file mode 100644 index 00000000..539c5e37 --- /dev/null +++ b/tests/pgpro688.py @@ -0,0 +1,45 @@ +import unittest +import os +import six +from .ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class ValidateTime(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ValidateTime, self).__init__(*args, **kwargs) + +# @classmethod +# def tearDownClass(cls): +# stop_all() + + def test_validate_recovery_time(self): + """ + make node with archiving, make backup, + get Recovery Time, validate to Recovery Time + EXPECT VALIDATE TO FAIL + Waiting PGPRO-688 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname), + set_archiving=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + id = self.backup_pb(node, backup_type='full') + recovery_time = self.show_pb(node, id=id)['recovery-time'] + + # Optional + #node.psql("postgres", "select pg_create_restore_point('123')") + #node.psql("postgres", "select txid_current()") + #node.psql("postgres", "select pg_switch_xlog()") + self.assertIn(six.b("INFO: backup validation completed successfully on"), + self.validate_pb(node, options=["--time='{0}'".format(recovery_time)])) + #### diff --git a/tests/pgpro702.py b/tests/pgpro702.py new file mode 100644 index 00000000..60361dd3 --- /dev/null +++ b/tests/pgpro702.py @@ -0,0 +1,39 @@ +import unittest +import os +import six +from .ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class ValidateTime(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ValidateTime, self).__init__(*args, **kwargs) + +# @classmethod +# def tearDownClass(cls): +# stop_all() + + def test_validate_recovery_time(self): + """ + make node without archiving, make stream backup, + get Recovery Time, validate to Recovery Time + EXPECT VALIDATE TO FAIL + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro702/{0}".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + id = self.backup_pb(node, backup_type='full', options=["--stream"]) + recovery_time = self.show_pb(node, id=id)['recovery-time'] + + self.assertIn(six.b("INFO: backup validation completed successfully on"), + self.validate_pb(node, options=["--time='{0}'".format(recovery_time)])) diff --git a/tests/ptrack_helpers.py b/tests/ptrack_helpers.py index 31b28e62..4c7184b7 100644 --- a/tests/ptrack_helpers.py +++ b/tests/ptrack_helpers.py @@ -292,17 +292,27 @@ class ProbackupTest(object): success = False self.assertEqual(success, True) - def run_pb(self, command): + def run_pb(self, command, async=False): try: # print [self.probackup_path] + command - output = subprocess.check_output( - [self.probackup_path] + command, - stderr=subprocess.STDOUT, - env=self.test_env - ) + if async is True: + return subprocess.Popen( + [self.probackup_path] + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ) if command[0] == 'backup': if '-q' in command or '--quiet' in command: return None + elif '-v' in command or '--verbose' in command: + return output else: # return backup ID return output.split()[2] @@ -322,7 +332,7 @@ class ProbackupTest(object): def clean_pb(self, node): shutil.rmtree(self.backup_dir(node), ignore_errors=True) - def backup_pb(self, node=None, data_dir=None, backup_dir=None, backup_type="full", options=[]): + def backup_pb(self, node=None, data_dir=None, backup_dir=None, backup_type="full", options=[], async=False): if data_dir is None: data_dir = node.data_dir if backup_dir is None: @@ -338,7 +348,7 @@ class ProbackupTest(object): if backup_type: cmd_list += ["-b", backup_type] - return self.run_pb(cmd_list + options) + return self.run_pb(cmd_list + options, async) def restore_pb(self, node=None, backup_dir=None, data_dir=None, id=None, options=[]): if data_dir is None: diff --git a/tests/restore_test.py b/tests/restore_test.py index ea35a0b1..f4e4e2b1 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -702,14 +702,66 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.restore_pb(node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]) # ) - - # Check tables + # Check tables node.start() count = node.execute("postgres", "SELECT count(*) FROM tbl") self.assertEqual(count[0][0], 4) - count = node.execute("postgres", "SELECT count(*) FROM tbl1") self.assertEqual(count[0][0], 4) - + node.stop() + + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + get Recovery Time, try to make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname), + set_archiving=True, + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + id = self.backup_pb(node, backup_type='full', options=["--stream"]) + recovery_time = self.show_pb(node, id=id)['recovery-time'] + + node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) + node.cleanup() + + self.restore_pb(node, options=['--time="{0}"'.format(recovery_time)]) + node.start({"-t": "600"}) + self.assertEqual(True, node.status()) + node.stop() + + def test_archive_node_backup_stream_additional_commit_pitr(self): + """ + make node with archiving, make stream backup, create table t_heap, + try to make pitr to Recovery Time, check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), + set_archiving=True, + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + id = self.backup_pb(node, backup_type='full', options=["--stream"]) + node.psql("postgres", "create table t_heap(a int)") + node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) + node.cleanup() + recovery_time = self.show_pb(node, id=id)['recovery-time'] + self.restore_pb(node, + options=["-j", "4", '--time="{0}"'.format(recovery_time)] + ) + node.start({"-t": "600"}) + + res = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in res[2]) node.stop() diff --git a/tests/validate_test.py b/tests/validate_test.py index 66082847..1c67b2fc 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -31,6 +31,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): ) node.start() + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -67,6 +68,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): with open(os.path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: backup_log.write(self.backup_pb(node, options=["--verbose"])) + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT,