From a5be84af44e5eee2ccd5ddec39618df53f2add7f Mon Sep 17 00:00:00 2001 From: "s.logvinenko" Date: Mon, 2 Oct 2017 14:53:24 +0300 Subject: [PATCH] Update tests --- tests/cfs_backup_enc.py | 686 +---------------------------------- tests/cfs_backup_noenc.py | 338 ++++++++++------- tests/cfs_restore_noenc.py | 178 ++++----- tests/helpers/__init__.py | 2 +- tests/helpers/cfs_helpers.py | 53 ++- 5 files changed, 334 insertions(+), 923 deletions(-) diff --git a/tests/cfs_backup_enc.py b/tests/cfs_backup_enc.py index 412e3a63..e190a1cc 100644 --- a/tests/cfs_backup_enc.py +++ b/tests/cfs_backup_enc.py @@ -1,688 +1,14 @@ import os -import unittest -from .helpers.cfs_helpers import find_by_extensions, find_by_name -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .cfs_backup_noenc import CfsBackupNoEncTest +from .helpers.cfs_helpers import find_by_name -module_name = 'cfs_backup_noenc' +module_name = 'cfs_backup_enc' tblspace_name = 'cfs_tblspace' -class CfsBackupEncTest(ProbackupTest, unittest.TestCase): - fname = None - backup_dir = None - node = None - -# --- Begin --- # +class CfsBackupEncTest(CfsBackupNoEncTest): + # --- Begin --- # def setUp(self): - global fname - global backup_dir - global node - os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'cfs_encryption': 'on', - 'max_wal_senders': '2' - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.start() - - self.create_tblspace_in_node(node, tblspace_name, True) - - tblspace = node.safe_psql( - "postgres", - "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) - ) - self.assertTrue( - tblspace_name in tblspace and "compression=true" in tblspace, - "ERROR: The tablespace not created or it create without compressions" - ) - - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - -# --- Section: Full --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace(self): - """ - Case: Check fullbackup empty compressed tablespace - """ - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_stream(self): - """ - Case: Check fullbackup empty compressed tablespace with options stream - """ - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table(self): - """ - Case: Make full backup after created table in the tablespace - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {0}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table_stream(self): - """ - Case: Make full backup after created table in the tablespace with option --stream - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - -# --- Section: Incremental from empty tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["Status"], - "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["Status"]) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - -# --- Section: Incremental from fill tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_after_create_table_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,25) i'.format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full)[0] - show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack)[0] - self.assertGreater( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"], - "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace (--stream). - Make ptrack backup after create table (--stream). - Check: incremental backup will not greater as full - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,25) i'.format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) - self.assertGreater( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"], - "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_after_create_table_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,25) i'.format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"], - "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_after_create_table_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace (--stream). - Make ptrack backup after create table (--stream). - Check: incremental backup will not greater as full - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,25) i'.format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"], - "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - -# --- end ---# - def tearDown(self): - node.cleanup() - self.del_test_dir(module_name, fname) + super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_backup_noenc.py b/tests/cfs_backup_noenc.py index 1d9ddfa8..8cd0cb7b 100644 --- a/tests/cfs_backup_noenc.py +++ b/tests/cfs_backup_noenc.py @@ -1,29 +1,22 @@ import os import unittest +import random -from .helpers.cfs_helpers import find_by_extensions, find_by_name +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_backup_enc' +module_name = 'cfs_backup_noenc' tblspace_name = 'cfs_tblspace' -class CfsBackupEncTest(ProbackupTest, unittest.TestCase): - fname = None - backup_dir = None - node = None - -# --- Begin --- # +class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): + # --- Begin --- # def setUp(self): - global fname - global backup_dir - global node + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -34,15 +27,15 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) - node.start() + self.node.start() - self.create_tblspace_in_node(node, tblspace_name, True) + self.create_tblspace_in_node(self.node, tblspace_name, True) - tblspace = node.safe_psql( + tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) ) @@ -52,11 +45,11 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) -# --- Section: Full --- # + # --- Section: Full --- # # @unittest.expectedFailure # @unittest.skip("skip") def test_fullbackup_empty_tablespace(self): @@ -66,7 +59,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -74,14 +67,14 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) @@ -94,7 +87,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -102,18 +95,18 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files found in backup dir" ) @@ -125,7 +118,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Case: Make full backup after created table in the tablespace """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -135,7 +128,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {0}".format( @@ -143,22 +136,22 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in {0}".format(os.path.join(backup_dir, 'node', backup_id)) + find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in {0}".format(os.path.join(self.backup_dir, 'node', backup_id)) ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) @@ -170,7 +163,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Case: Make full backup after created table in the tablespace with option --stream """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -180,7 +173,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -188,26 +181,26 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) -# --- Section: Incremental from empty tablespace --- # + # --- Section: Incremental from empty tablespace --- # # @unittest.expectedFailure # @unittest.skip("skip") def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): @@ -217,7 +210,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ try: - self.backup_node(backup_dir, 'node', node, backup_type='full') + self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -226,7 +219,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -236,7 +229,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='ptrack') except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -244,22 +237,22 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) @@ -272,7 +265,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ try: - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -281,7 +274,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -291,7 +284,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='ptrack', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -299,22 +292,22 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) @@ -327,7 +320,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ try: - self.backup_node(backup_dir, 'node', node, backup_type='full') + self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -336,7 +329,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -346,7 +339,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page') except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -354,22 +347,22 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) @@ -382,7 +375,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ try: - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -391,7 +384,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -401,7 +394,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -409,26 +402,26 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) -# --- Section: Incremental from fill tablespace --- # + # --- Section: Incremental from fill tablespace --- # # @unittest.expectedFailure # @unittest.skip("skip") def test_fullbackup_after_create_table_ptrack_after_create_table(self): @@ -438,7 +431,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -448,7 +441,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_full = None try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') + backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -457,7 +450,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -467,7 +460,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_ptrack = None try: - backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + backup_id_ptrack = self.backup_node(self.backup_dir, 'node', self.node, backup_type='ptrack') except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -476,8 +469,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) + show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb(self.backup_dir, 'node', backup_id_ptrack) self.assertGreater( show_backup_ptrack["data-bytes"], show_backup_full["data-bytes"], @@ -491,12 +484,12 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): """ - Case: Make full backup before created table in the tablespace (--stream). - Make ptrack backup after create table (--stream). + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). Check: incremental backup will not greater as full """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -506,7 +499,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_full = None try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -515,7 +508,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -525,7 +518,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_ptrack = None try: - backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + backup_id_ptrack = self.backup_node(self.backup_dir, 'node', self.node, backup_type='ptrack', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -534,8 +527,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) + show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb(self.backup_dir, 'node', backup_id_ptrack) self.assertGreater( show_backup_ptrack["data-bytes"], show_backup_full["data-bytes"], @@ -554,7 +547,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -564,7 +557,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_full = None try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') + backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -573,7 +566,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -583,7 +576,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_page = None try: - backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_page = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page') except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -592,8 +585,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) + show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb(self.backup_dir, 'node', backup_id_page) self.assertGreater( show_backup_page["data-bytes"], show_backup_full["data-bytes"], @@ -607,12 +600,12 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_fullbackup_after_create_table_page_after_create_table_stream(self): """ - Case: Make full backup before created table in the tablespace (--stream). - Make ptrack backup after create table (--stream). + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). Check: incremental backup will not greater as full """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -622,7 +615,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_full = None try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -631,7 +624,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -641,7 +634,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_page = None try: - backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) + backup_id_page = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -650,8 +643,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) + show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb(self.backup_dir, 'node', backup_id_page) self.assertGreater( show_backup_page["data-bytes"], show_backup_full["data-bytes"], @@ -661,43 +654,130 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) -# --- Make backup with not valid data (broken .cfm) --- # + # --- Make backup with not valid data(broken .cfm) --- # # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_delete_random_cfm_file_from_tablespace_dir(self): - pass + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions([self.get_tblspace_path(self.node,tblspace_name)],['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + os.remove(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) # @unittest.expectedFailure # @unittest.skip("skip") def test_delete_file_pg_compression_from_tablespace_dir(self): - os.remove(find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression'])[0]) + os.remove(find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression'])[0]) self.assertRaises( ProbackupException, - self.backup_node,backup_dir, 'node', node, backup_type='full' + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_delete_random_data_file_from_tablespace_dir(self): - pass + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern([self.get_tblspace_path(self.node,tblspace_name)],'^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + os.remove(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_broken_random_cfm_file_into_tablespace_dir(self): - pass + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions([self.get_tblspace_path(self.node,tblspace_name)],['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + corrupt_file(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_broken_random_data_file_into_tablespace_dir(self): - pass + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern([self.get_tblspace_path(self.node,tblspace_name)],'^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + corrupt_file(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_broken_file_pg_compression_into_tablespace_dir(self): - pass -# --- Validation backup --- # + corrupted_file = find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression'])[0] + + self.assertTrue( + corrupt_file(corrupted_file), + "ERROR: File is not corrupted or it missing" + ) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) + + # --- Validation backup --- # # @unittest.expectedFailure @unittest.skip("skip") def test_delete_random_cfm_file_from_backup_dir(self): @@ -706,8 +786,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_delete_file_pg_compression_from_backup_dir(self): - self.backup_node(backup_dir, 'node', node, backup_type = 'full') - show_backup = self.show_pb(backup_dir,'node')[0] + self.backup_node(self.backup_dir, 'node', self.node, backup_type = 'full') + show_backup = self.show_pb(self.backup_dir,'node')[0] self.assertEqual( "OK", @@ -715,9 +795,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): "ERROR: Backup is not valid. \n Backup status: %s" % show_backup["Status"] ) - os.remove(find_by_name([os.path.join(backup_dir, 'backups', 'node', show_backup["ID"])], ['pg_compression'])[0]) + os.remove(find_by_name([os.path.join(self.backup_dir, 'backups', 'node', show_backup["ID"])], ['pg_compression'])[0]) - self.assertRaises(ProbackupException, self.validate_pb, backup_dir, 'node') + self.assertRaises(ProbackupException, self.validate_pb, self.backup_dir, 'node') # @unittest.expectedFailure @unittest.skip("skip") @@ -739,7 +819,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): def test_broken_file_pg_compression_into_backup_dir(self): pass -# --- end ---# + # --- End ---# def tearDown(self): - node.cleanup() - self.del_test_dir(module_name, fname) + self.node.cleanup() + self.del_test_dir(module_name, self.fname) diff --git a/tests/cfs_restore_noenc.py b/tests/cfs_restore_noenc.py index 1e698f57..0af9f7b7 100644 --- a/tests/cfs_restore_noenc.py +++ b/tests/cfs_restore_noenc.py @@ -22,21 +22,13 @@ tblspace_name = 'cfs_tblspace_noenc' tblspace_name_new = 'cfs_tblspace_new' -class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): - fname = None - backup_dir = None - node = None - +class CfsRestoreBaseTest(ProbackupTest, unittest.TestCase): def setUp(self): - global fname - global backup_dir - global node + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -47,16 +39,18 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) - node.start() + self.node.start() - self.create_tblspace_in_node(node, tblspace_name, True) + self.add_data_in_cluster() + + self.create_tblspace_in_node(self.node, tblspace_name, True) self.backup_id = None try: - self.backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed \n {0} \n {1}".format( @@ -65,18 +59,27 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): ) ) + def add_data_in_cluster(self): + pass + + def tearDown(self): + self.node.cleanup() + self.del_test_dir(module_name, self.fname) + + +class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBaseTest): # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_empty_tablespace_from_fullbackup(self): """ Case: Restore empty tablespace from valid full backup. """ - node.stop({"-m": "immediate"}) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop({"-m": "immediate"}) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) try: - self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id) + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) except ProbackupException as e: self.fail( "ERROR: Restore failed. \n {0} \n {1}".format( @@ -85,12 +88,12 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ["pg_compression"]), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." ) try: - node.start() + self.node.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -98,7 +101,7 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - tblspace = node.safe_psql( + tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) ) @@ -107,79 +110,34 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): "ERROR: The tablespace not restored or it restored without compressions" ) - def tearDown(self): - node.cleanup() - self.del_test_dir(module_name, fname) - -class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): - fname = None - backup_dir = None - node = None - - def setUp(self): - global fname - global backup_dir - global node - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'cfs_encryption': 'off', - 'max_wal_senders': '2' - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.start() - - self.create_tblspace_in_node(node, tblspace_name, True) - - node.safe_psql( +class CfsRestoreNoencTest(CfsRestoreBaseTest): + def add_data_in_cluster(self): + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ MD5(repeat(i::text,10))::tsvector AS tsvector \ FROM generate_series(0,1e5) i'.format('t1', tblspace_name) ) - self.table_t1 = node.safe_psql( + self.table_t1 = self.node.safe_psql( "postgres", "SELECT * FROM t1" ) - self.backup_id = None - try: - self.backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) -# --- Restore from full backup ---# + # --- Restore from full backup ---# # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_from_fullbackup_to_old_location(self): """ Case: Restore instance from valid full backup to old location. """ - node.stop(['-m', 'immediate']) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) try: - self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id) + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -188,11 +146,11 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) try: - node.start() + self.node.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -202,7 +160,7 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) self.assertEqual( - repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), repr(self.table_t1) ) @@ -212,12 +170,12 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): """ Case: Restore instance from valid full backup to old location. """ - node.stop(['-m', 'immediate']) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) try: - self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id, options=['-j', '3']) + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -226,11 +184,11 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node,tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) try: - node.start() + self.node.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -240,7 +198,7 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) self.assertEqual( - repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), repr(self.table_t1) ) @@ -250,14 +208,14 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): """ Case: Restore instance from valid full backup to new location. """ - node.stop(['-m', 'immediate']) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, fname)) + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) try: - self.restore_node(backup_dir, 'node', node_new, backup_id=self.backup_id) + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -266,11 +224,11 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node_new,tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node_new, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) try: - node_new.start() + self.node_new.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -280,10 +238,10 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) self.assertEqual( - repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), repr(self.table_t1) ) - node_new.cleanup() + self.node_new.cleanup() # @unittest.expectedFailure # @unittest.skip("skip") @@ -291,14 +249,14 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): """ Case: Restore instance from valid full backup to new location. """ - node.stop(['-m', 'immediate']) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, fname)) + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) try: - self.restore_node(backup_dir, 'node', node_new, backup_id=self.backup_id, options=['-j', '5']) + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id, options=['-j', '5']) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -307,11 +265,11 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node_new,tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node_new, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) try: - node_new.start() + self.node_new.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -321,10 +279,10 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) self.assertEqual( - repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), repr(self.table_t1) ) - node_new.cleanup() + self.node_new.cleanup() # @unittest.expectedFailure @unittest.skip("skip") @@ -391,7 +349,3 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): Case: Restore from backup to new location, four jobs """ pass - - def tearDown(self): - node.cleanup() - self.del_test_dir(module_name, fname) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index 769446ad..ac64c423 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['ptrack_helpers', 'expected_errors'] +__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] #from . import * \ No newline at end of file diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py index d35050f1..67e2b331 100644 --- a/tests/helpers/cfs_helpers.py +++ b/tests/helpers/cfs_helpers.py @@ -1,4 +1,8 @@ import os +import re +import random +import string + def find_by_extensions(dirs=None, extensions=None): """ @@ -22,6 +26,30 @@ def find_by_extensions(dirs=None, extensions=None): return files + +def find_by_pattern(dirs=None, pattern=None): + """ + find_by_pattern(['path1','path2'],'^.*/*.txt') + :return: + Return list of files include full path by pattern + """ + files = [] + new_dirs = [] + + if dirs is not None and pattern is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if re.match(pattern,d): + files.append(d) + + if new_dirs: + files.extend(find_by_pattern(new_dirs, pattern)) + + return files + + def find_by_name(dirs=None, filename=None): files = [] new_dirs = [] @@ -37,4 +65,27 @@ def find_by_name(dirs=None, filename=None): if new_dirs: files.extend(find_by_name(new_dirs, filename)) - return files \ No newline at end of file + return files + + +def corrupt_file(filename): + file_size = None + try: + file_size = os.path.getsize(filename) + except OSError: + return False + + try: + with open(filename, "rb+") as f: + f.seek(random.randint(int(0.1*file_size),int(0.8*file_size))) + f.write(random_string(0.1*file_size)) + f.close() + except OSError: + return False + + return True + + +def random_string(n): + a = string.ascii_letters + string.digits + return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file