From 2bd2b6e1b8a10703113652efb2507aecf866a5a9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 11 Oct 2017 18:08:56 +0300 Subject: [PATCH] tests fixes --- tests/Readme.md | 11 +++++++ tests/__init__.py | 3 ++ tests/archive.py | 2 +- tests/helpers/ptrack_helpers.py | 15 ++++++++- tests/ptrack.py | 55 ++++++++++++++++++++++----------- tests/validate_test.py | 2 +- 6 files changed, 67 insertions(+), 21 deletions(-) create mode 100644 tests/Readme.md diff --git a/tests/Readme.md b/tests/Readme.md new file mode 100644 index 00000000..0ccfaf1c --- /dev/null +++ b/tests/Readme.md @@ -0,0 +1,11 @@ +[см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) + +``` +Note: For now there are tests only for Linix +``` + +``` +export PG_CONFIG=/path/to/pg_config +export PGPRO_PARANOIA_MODE=ON/OFF +python -m unittest [-v] tests +``` diff --git a/tests/__init__.py b/tests/__init__.py index 83e2f723..50881635 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -52,3 +52,6 @@ def load_tests(loader, tests, pattern): # ptrack backup on replica should work correctly # archive: # immediate recovery and full recovery +# 10vanilla_1.3ptrack + +# 10vanilla+ +# 9.6vanilla_1.3ptrack + diff --git a/tests/archive.py b/tests/archive.py index abbb27f3..61744408 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -233,7 +233,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): os.remove(file) sleep(5) - node.safe_psql('postgres', 'select pg_switch_wal()') + self.switch_wal_segment(node) with open(log_file, 'r') as f: log_content = f.read() diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 9750c804..056d51d4 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -4,7 +4,7 @@ from sys import exit, argv, version_info import subprocess import shutil import six -from testgres import get_new_node, clean_all +from testgres import get_new_node, clean_all, version_to_num import hashlib import re import pwd @@ -123,6 +123,12 @@ class ProbackupTest(object): self.test_env["LC_MESSAGES"] = "C" self.test_env["LC_TIME"] = "C" + + self.paranoia = False + if 'PGPRO_PARANOIA_MODE' in self.test_env: + if self.test_env['PGPRO_PARANOIA_MODE'] == 'ON': + self.paranoia = True + self.helpers_path = os.path.dirname(os.path.realpath(__file__)) self.dir_path = os.path.abspath(os.path.join(self.helpers_path, os.pardir)) self.tmp_path = os.path.abspath(os.path.join(self.dir_path, 'tmp_dirs')) @@ -582,6 +588,13 @@ class ProbackupTest(object): """ Returns current user name """ return pwd.getpwuid(os.getuid())[0] + def switch_wal_segment(self, node): + """ Execute pg_switch_wal/xlog() in given node""" + if version_to_num(node.safe_psql("postgres", "show server_version")) >= version_to_num('10.0'): + node.safe_psql("postgres", "select pg_switch_wal()") + else: + node.safe_psql("postgres", "select pg_switch_xlog()") + def del_test_dir(self, module_name, fname): """ Del testdir and optimistically try to del module dir""" try: diff --git a/tests/ptrack.py b/tests/ptrack.py index c31770eb..6e0a6744 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -140,13 +140,15 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'node', node, backup_id=ptrack_backup_id, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) pgdata_restored = self.pgdata_content(node.data_dir) - # self.compare_pgdata(pgdata, pgdata_restored) node.start() while node.safe_psql("postgres", "select pg_is_in_recovery()") == 't\n': time.sleep(1) ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -203,12 +205,15 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'node', node, backup_id=ptrack_backup_id, options=["-j", "4", "--time={0}".format(ptrack_target_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) pgdata_restored = self.pgdata_content(node.data_dir) - # self.compare_pgdata(pgdata, pgdata_restored) node.start() while node.safe_psql("postgres", "select pg_is_in_recovery()") == 't\n': time.sleep(1) ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + node.cleanup() # Clean after yourself @@ -401,8 +406,9 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): node_restored.cleanup() # COMPARE PHYSICAL CONTENT self.restore_node(backup_dir, 'node', node_restored, backup_id=backup_id, options=["-j", "4"]) - pgdata_new = self.pgdata_content(node_restored.data_dir) - # self.compare_pgdata(pgdata, pgdata_restored) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE node_restored.append_conf("postgresql.auto.conf", "port = {0}".format(node_restored.port)) @@ -424,8 +430,9 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): node_restored.start() # COMPARE PHYSICAL CONTENT - pgdata_new = self.pgdata_content(node_restored.data_dir) - # self.compare_pgdata(pgdata, pgdata_restored) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) try: node_restored.safe_psql('db1', 'select 1') @@ -485,8 +492,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): ]) # GET RESTORED PGDATA AND COMPARE - pgdata_new = self.pgdata_content(node_restored.data_dir) - # self.compare_pgdata(pgdata, pgdata_new) + pgdata_restored = self.pgdata_content(node_restored.data_dir) # START RESTORED NODE node_restored.append_conf('postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) node_restored.start() @@ -496,6 +502,8 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): result_new = node_restored.safe_psql("postgres", "select * from t_heap") self.assertEqual(result, result_new, 'lost some data after restore') + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -538,9 +546,10 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): "-T", "{0}={1}".format(self.get_tblspace_path(node,'somedata'), self.get_tblspace_path(node_restored,'somedata'))]) # GET PHYSICAL CONTENT - pgdata_new = self.pgdata_content(node_restored.data_dir) + pgdata_restored = self.pgdata_content(node_restored.data_dir) # COMPARE PHYSICAL CONTENT - # self.compare_pgdata(pgdata, pgdata_restored) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE node_restored.start() @@ -649,9 +658,8 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) result = node.safe_psql("postgres", "select * from t_heap") - pgdata_new = self.pgdata_content(restored_node.data_dir) - # COMPARE PHYSICAL CONTENT - # self.compare_pgdata(pgdata, pgdata_restored) + pgdata_restored = self.pgdata_content(restored_node.data_dir) + # START RESTORED NODE restored_node.append_conf("postgresql.auto.conf", "port = {0}".format(restored_node.port)) restored_node.start() @@ -660,6 +668,11 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): result_new = restored_node.safe_psql("postgres", "select * from t_heap") self.assertEqual(result, result_new) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + restored_node.cleanup() shutil.rmtree(tblspc_path_new, ignore_errors=True) @@ -675,9 +688,8 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) restored_node.append_conf("postgresql.auto.conf", "port = {0}".format(restored_node.port)) - # COMPARE PHYSICAL CONTENT - pgdata_new = self.pgdata_content(restored_node.data_dir) - # self.compare_pgdata(pgdata, pgdata_restored) + # GET PHYSICAL CONTENT + pgdata_restored = self.pgdata_content(restored_node.data_dir) # START RESTORED NODE restored_node.start() @@ -687,13 +699,16 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): result_new = restored_node.safe_psql("postgres", "select * from t_heap") self.assertEqual(result, result_new) + if self.paranoia: + # COMPARE PHYSICAL CONTENT + self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_relation_with_multiple_segments(self): """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" - self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), @@ -736,7 +751,6 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM NODE_RESTORED pgdata_restored = self.pgdata_content(restored_node.data_dir) - # self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf("postgresql.auto.conf", "port = {0}".format(restored_node.port)) @@ -745,7 +759,12 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase): time.sleep(1) result_new = restored_node.safe_psql("postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES self.assertEqual(result, result_new) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py index cb978164..a1f19626 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -76,7 +76,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() target_xid = res[0][0] - node.execute("postgres", "SELECT pg_switch_wal()") + self.switch_wal_segment(node) self.assertIn("INFO: backup validation completed successfully", self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]),