From c4b382b0d59501e376083a669d73dcd863675a29 Mon Sep 17 00:00:00 2001 From: sfalkon Date: Thu, 28 Sep 2017 21:09:35 +0300 Subject: [PATCH 1/7] Add cfs_backup tests --- tests/__init__.py | 5 +- tests/cfs_backup_enc.py | 715 ++++++++++++++++++++++++++++++++ tests/cfs_backup_noenc.py | 775 +++++++++++++++++++++++++++++++++++ tests/helpers/cfs_helpers.py | 40 ++ 4 files changed, 1534 insertions(+), 1 deletion(-) create mode 100644 tests/cfs_backup_enc.py create mode 100644 tests/cfs_backup_noenc.py create mode 100644 tests/helpers/cfs_helpers.py diff --git a/tests/__init__.py b/tests/__init__.py index 66db8bcd..e613e455 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,7 +6,8 @@ from . import init_test, option_test, show_test, \ ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ - false_positive, replica, compression, page, ptrack, archive + false_positive, replica, compression, page, ptrack, archive, \ + cfs_backup_noenc, cfs_backup_enc def load_tests(loader, tests, pattern): @@ -36,6 +37,8 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(compression)) suite.addTests(loader.loadTestsFromModule(page)) suite.addTests(loader.loadTestsFromModule(archive)) + suite.addTests(loader.loadTestsFromModule(cfs_backup_noenc)) + suite.addTests(loader.loadTestsFromModule(cfs_backup_enc)) return suite diff --git a/tests/cfs_backup_enc.py b/tests/cfs_backup_enc.py new file mode 100644 index 00000000..ae4deed7 --- /dev/null +++ b/tests/cfs_backup_enc.py @@ -0,0 +1,715 @@ +import os +import unittest + +from .helpers.cfs_helpers import find_by_extensions, find_by_name +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_backup_noenc' +tblspace_name = 'cfs_tblspace' + + +class CfsBackupEncTest(ProbackupTest, unittest.TestCase): + fname = None + backup_dir = None + node = None + +# --- Begin --- # + def setUp(self): + global fname + global backup_dir + global node + + os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'cfs_encryption': 'on', + 'max_wal_senders': '2' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + +# --- Section: Prepare --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_create_tblspace_compression(self): + """ + Case: Check to create compression tablespace + """ + self.create_tblspace_in_node(node, tblspace_name, True) + tblspace = node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) + ) + node.safe_psql( + "postgres", + 'create table {0} tablespace {1} \ + as select i as id, md5(i::text) as text, \ + md5(repeat(i::text,10))::tsvector as tsvector \ + from generate_series(0,256) i'.format('t1', tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not created or it create without compressions" + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + +# --- Section: Full --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace(self): + """ + Case: Check fullbackup empty compressed tablespace + """ + self.create_tblspace_in_node(node, tblspace_name, True) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_stream(self): + """ + Case: Check fullbackup empty compressed tablespace with options stream + """ + self.create_tblspace_in_node(node, tblspace_name, True) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table(self): + """ + Case: Make full backup after created table in the tablespace + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {0}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table_stream(self): + """ + Case: Make full backup after created table in the tablespace with option --stream + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + +# --- Section: Incremental from empty tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + try: + self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + try: + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + try: + self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + try: + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["Status"], + "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["Status"]) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + +# --- Section: Incremental from fill tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_after_create_table_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,25) i'.format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full)[0] + show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack)[0] + self.assertGreater( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"], + "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace (--stream). + Make ptrack backup after create table (--stream). + Check: incremental backup will not greater as full + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,25) i'.format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"], + "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_after_create_table_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,25) i'.format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"], + "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_after_create_table_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace (--stream). + Make ptrack backup after create table (--stream). + Check: incremental backup will not greater as full + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,25) i'.format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"], + "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + +# --- end ---# + def tearDown(self): + node.cleanup() + self.del_test_dir(module_name, fname) diff --git a/tests/cfs_backup_noenc.py b/tests/cfs_backup_noenc.py new file mode 100644 index 00000000..af1bf119 --- /dev/null +++ b/tests/cfs_backup_noenc.py @@ -0,0 +1,775 @@ +import os +import unittest + +from .helpers.cfs_helpers import find_by_extensions, find_by_name +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_backup_enc' +tblspace_name = 'cfs_tblspace' + + +class CfsBackupEncTest(ProbackupTest, unittest.TestCase): + fname = None + backup_dir = None + node = None + +# --- Begin --- # + def setUp(self): + global fname + global backup_dir + global node + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + +# --- Section: Prepare --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_create_tblspace_compression(self): + """ + Case: Check to create compression tablespace + """ + self.create_tblspace_in_node(node, tblspace_name, True) + tblspace = node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) + ) + node.safe_psql( + "postgres", + 'create table {0} tablespace {1} \ + as select i as id, md5(i::text) as text, \ + md5(repeat(i::text,10))::tsvector as tsvector \ + from generate_series(0,256) i'.format('t1', tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not created or it create without compressions" + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + +# --- Section: Full --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace(self): + """ + Case: Check fullbackup empty compressed tablespace + """ + self.create_tblspace_in_node(node, tblspace_name, True) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_stream(self): + """ + Case: Check fullbackup empty compressed tablespace with options stream + """ + self.create_tblspace_in_node(node, tblspace_name, True) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table(self): + """ + Case: Make full backup after created table in the tablespace + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {0}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in {0}".format(os.path.join(backup_dir, 'node', backup_id)) + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table_stream(self): + """ + Case: Make full backup after created table in the tablespace with option --stream + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + +# --- Section: Incremental from empty tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + try: + self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + try: + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + try: + self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + try: + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + show_backup = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), + "ERROR: .cmf files not found in backup dir" + ) + self.assertFalse( + find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + +# --- Section: Incremental from fill tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_after_create_table_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,25) i'.format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"], + "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace (--stream). + Make ptrack backup after create table (--stream). + Check: incremental backup will not greater as full + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,25) i'.format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"], + "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_after_create_table_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,25) i'.format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page') + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"], + "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_fullbackup_after_create_table_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace (--stream). + Make ptrack backup after create table (--stream). + Check: incremental backup will not greater as full + """ + + self.create_tblspace_in_node(node, tblspace_name, True) + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Full backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,25) i'.format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) + except ProbackupException as e: + self.assertTrue( + False, + "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( + str(e.cmd), + str(e.message) + ) + ) + + show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"], + "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + +# --- Make backup with not valid data (broken .cfm) --- # + # @unittest.expectedFailure + @unittest.skip("skip") + def test_delete_random_cfm_file_from_tablespace_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_delete_file_pg_compression_from_tablespace_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_delete_random_data_file_from_tablespace_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_broken_random_cfm_file_into_tablespace_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_broken_random_data_file_into_tablespace_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_broken_file_pg_compression_into_tablespace_dir(self): + pass + +# --- Validation backup --- # + # @unittest.expectedFailure + @unittest.skip("skip") + def test_delete_random_cfm_file_from_backup_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_delete_file_pg_compression_from_backup_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_delete_random_data_file_from_backup_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_broken_random_cfm_file_into_backup_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_broken_random_data_file_into_backup_dir(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_broken_file_pg_compression_into_backup_dir(self): + pass + +# --- end ---# + def tearDown(self): + node.cleanup() + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py new file mode 100644 index 00000000..d35050f1 --- /dev/null +++ b/tests/helpers/cfs_helpers.py @@ -0,0 +1,40 @@ +import os + +def find_by_extensions(dirs=None, extensions=None): + """ + find_by_extensions(['path1','path2'],['.txt','.log']) + :return: + Return list of files include full path by file extensions + """ + files = [] + new_dirs = [] + + if dirs is not None and extensions is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.splitext(d)[1] in extensions: + files.append(d) + + if new_dirs: + files.extend(find_by_extensions(new_dirs, extensions)) + + return files + +def find_by_name(dirs=None, filename=None): + files = [] + new_dirs = [] + + if dirs is not None and filename is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.basename(d) in filename: + files.append(d) + + if new_dirs: + files.extend(find_by_name(new_dirs, filename)) + + return files \ No newline at end of file From 8400cf44c1ea683fbba81f862b2bf154efdcd649 Mon Sep 17 00:00:00 2001 From: sfalkon Date: Fri, 29 Sep 2017 13:48:02 +0300 Subject: [PATCH 2/7] Add cfs_restore_noenc. Update cfs_backup_enc and cfs_backup_noenc --- tests/__init__.py | 3 +- tests/cfs_backup_enc.py | 153 ++++++-------- tests/cfs_backup_noenc.py | 234 ++++++++++------------ tests/cfs_restore_noenc.py | 397 +++++++++++++++++++++++++++++++++++++ 4 files changed, 564 insertions(+), 223 deletions(-) create mode 100644 tests/cfs_restore_noenc.py diff --git a/tests/__init__.py b/tests/__init__.py index e613e455..21ee595a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,7 +7,7 @@ from . import init_test, option_test, show_test, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ - cfs_backup_noenc, cfs_backup_enc + cfs_backup_noenc, cfs_backup_enc, cfs_restore_noenc def load_tests(loader, tests, pattern): @@ -39,6 +39,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(cfs_backup_noenc)) suite.addTests(loader.loadTestsFromModule(cfs_backup_enc)) + suite.addTests(loader.loadTestsFromModule(cfs_restore_noenc)) return suite diff --git a/tests/cfs_backup_enc.py b/tests/cfs_backup_enc.py index ae4deed7..412e3a63 100644 --- a/tests/cfs_backup_enc.py +++ b/tests/cfs_backup_enc.py @@ -42,29 +42,17 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): node.start() -# --- Section: Prepare --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_create_tblspace_compression(self): - """ - Case: Check to create compression tablespace - """ self.create_tblspace_in_node(node, tblspace_name, True) + tblspace = node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) ) - node.safe_psql( - "postgres", - 'create table {0} tablespace {1} \ - as select i as id, md5(i::text) as text, \ - md5(repeat(i::text,10))::tsvector as tsvector \ - from generate_series(0,256) i'.format('t1', tblspace_name) - ) self.assertTrue( tblspace_name in tblspace and "compression=true" in tblspace, "ERROR: The tablespace not created or it create without compressions" ) + self.assertTrue( find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" @@ -77,7 +65,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ Case: Check fullbackup empty compressed tablespace """ - self.create_tblspace_in_node(node, tblspace_name, True) backup_id = None try: @@ -85,9 +72,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -107,7 +94,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ Case: Check fullbackup empty compressed tablespace with options stream """ - self.create_tblspace_in_node(node, tblspace_name, True) backup_id = None try: @@ -115,9 +101,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -143,8 +129,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Case: Make full backup after created table in the tablespace """ - self.create_tblspace_in_node(node, tblspace_name, True) - node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -159,9 +143,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {0}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {0}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -191,8 +175,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Case: Make full backup after created table in the tablespace with option --stream """ - self.create_tblspace_in_node(node, tblspace_name, True) - node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -205,11 +187,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -240,15 +221,14 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Make ptrack backup after create table """ - self.create_tblspace_in_node(node, tblspace_name, True) try: self.backup_node(backup_dir, 'node', node, backup_type='full') except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -266,9 +246,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -298,15 +278,14 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Make ptrack backup after create table """ - self.create_tblspace_in_node(node, tblspace_name, True) try: self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -324,9 +303,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -356,15 +335,14 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Make page backup after create table """ - self.create_tblspace_in_node(node, tblspace_name, True) try: self.backup_node(backup_dir, 'node', node, backup_type='full') except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -382,9 +360,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -414,15 +392,14 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Make page backup after create table """ - self.create_tblspace_in_node(node, tblspace_name, True) try: self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -440,9 +417,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -474,7 +451,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - self.create_tblspace_in_node(node, tblspace_name, True) node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -489,9 +465,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -509,9 +485,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -535,7 +511,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - self.create_tblspace_in_node(node, tblspace_name, True) node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -550,9 +525,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -570,9 +545,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -596,7 +571,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - self.create_tblspace_in_node(node, tblspace_name, True) node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -611,9 +585,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -631,9 +605,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -657,7 +631,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - self.create_tblspace_in_node(node, tblspace_name, True) node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -672,9 +645,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -692,9 +665,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: self.assertTrue( False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) diff --git a/tests/cfs_backup_noenc.py b/tests/cfs_backup_noenc.py index af1bf119..1d9ddfa8 100644 --- a/tests/cfs_backup_noenc.py +++ b/tests/cfs_backup_noenc.py @@ -40,29 +40,17 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): node.start() -# --- Section: Prepare --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_create_tblspace_compression(self): - """ - Case: Check to create compression tablespace - """ self.create_tblspace_in_node(node, tblspace_name, True) + tblspace = node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) ) - node.safe_psql( - "postgres", - 'create table {0} tablespace {1} \ - as select i as id, md5(i::text) as text, \ - md5(repeat(i::text,10))::tsvector as tsvector \ - from generate_series(0,256) i'.format('t1', tblspace_name) - ) self.assertTrue( tblspace_name in tblspace and "compression=true" in tblspace, "ERROR: The tablespace not created or it create without compressions" ) + self.assertTrue( find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" @@ -75,17 +63,15 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ Case: Check fullbackup empty compressed tablespace """ - self.create_tblspace_in_node(node, tblspace_name, True) backup_id = None try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -105,17 +91,15 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ Case: Check fullbackup empty compressed tablespace with options stream """ - self.create_tblspace_in_node(node, tblspace_name, True) backup_id = None try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -141,8 +125,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Case: Make full backup after created table in the tablespace """ - self.create_tblspace_in_node(node, tblspace_name, True) - node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -155,11 +137,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {0}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {0}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -189,8 +170,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Case: Make full backup after created table in the tablespace with option --stream """ - self.create_tblspace_in_node(node, tblspace_name, True) - node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -203,11 +182,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -238,15 +216,13 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Make ptrack backup after create table """ - self.create_tblspace_in_node(node, tblspace_name, True) try: self.backup_node(backup_dir, 'node', node, backup_type='full') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -262,11 +238,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -296,15 +271,13 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Make ptrack backup after create table """ - self.create_tblspace_in_node(node, tblspace_name, True) try: self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -320,11 +293,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -354,15 +326,13 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Make page backup after create table """ - self.create_tblspace_in_node(node, tblspace_name, True) try: self.backup_node(backup_dir, 'node', node, backup_type='full') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -378,11 +348,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -412,15 +381,13 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Make page backup after create table """ - self.create_tblspace_in_node(node, tblspace_name, True) try: self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -436,11 +403,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) show_backup = self.show_pb(backup_dir, 'node', backup_id) @@ -472,7 +438,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - self.create_tblspace_in_node(node, tblspace_name, True) node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -485,11 +450,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -505,11 +469,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -533,7 +496,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - self.create_tblspace_in_node(node, tblspace_name, True) node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -546,11 +508,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -566,11 +527,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -594,7 +554,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - self.create_tblspace_in_node(node, tblspace_name, True) node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -607,11 +566,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -627,11 +585,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page') except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -655,7 +612,6 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - self.create_tblspace_in_node(node, tblspace_name, True) node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ @@ -668,11 +624,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -688,11 +643,10 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): try: backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup wasn't created.\n {0} \n {1}".format( - str(e.cmd), - str(e.message) + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) ) ) @@ -714,9 +668,14 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): pass # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_delete_file_pg_compression_from_tablespace_dir(self): - pass + os.remove(find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression'])[0]) + + self.assertRaises( + ProbackupException, + self.backup_node,backup_dir, 'node', node, backup_type='full' + ) # @unittest.expectedFailure @unittest.skip("skip") @@ -745,9 +704,20 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): pass # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_delete_file_pg_compression_from_backup_dir(self): - pass + self.backup_node(backup_dir, 'node', node, backup_type = 'full') + show_backup = self.show_pb(backup_dir,'node')[0] + + self.assertEqual( + "OK", + show_backup["Status"], + "ERROR: Backup is not valid. \n Backup status: %s" % show_backup["Status"] + ) + + os.remove(find_by_name([os.path.join(backup_dir, 'backups', 'node', show_backup["ID"])], ['pg_compression'])[0]) + + self.assertRaises(ProbackupException, self.validate_pb, backup_dir, 'node') # @unittest.expectedFailure @unittest.skip("skip") diff --git a/tests/cfs_restore_noenc.py b/tests/cfs_restore_noenc.py new file mode 100644 index 00000000..1e698f57 --- /dev/null +++ b/tests/cfs_restore_noenc.py @@ -0,0 +1,397 @@ +""" +restore + Syntax: + + pg_probackup restore -B backupdir --instance instance_name + [-D datadir] + [ -i backup_id | [{--time=time | --xid=xid } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] + [-j num_threads] [--progress] [-q] [-v] + +""" +import os +import unittest +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'cfs_restore_noenc' + +tblspace_name = 'cfs_tblspace_noenc' +tblspace_name_new = 'cfs_tblspace_new' + + +class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): + fname = None + backup_dir = None + node = None + + def setUp(self): + global fname + global backup_dir + global node + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + + self.create_tblspace_in_node(node, tblspace_name, True) + self.backup_id = None + try: + self.backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_empty_tablespace_from_fullbackup(self): + """ + Case: Restore empty tablespace from valid full backup. + """ + node.stop({"-m": "immediate"}) + node.cleanup() + shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + + try: + self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ["pg_compression"]), + "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." + ) + + try: + node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + tblspace = node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not restored or it restored without compressions" + ) + + def tearDown(self): + node.cleanup() + self.del_test_dir(module_name, fname) + + +class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): + fname = None + backup_dir = None + node = None + + def setUp(self): + global fname + global backup_dir + global node + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + + self.create_tblspace_in_node(node, tblspace_name, True) + + node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,1e5) i'.format('t1', tblspace_name) + ) + self.table_t1 = node.safe_psql( + "postgres", + "SELECT * FROM t1" + ) + self.backup_id = None + try: + self.backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + +# --- Restore from full backup ---# + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location(self): + """ + Case: Restore instance from valid full backup to old location. + """ + node.stop(['-m', 'immediate']) + node.cleanup() + shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + + try: + self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_3_jobs(self): + """ + Case: Restore instance from valid full backup to old location. + """ + node.stop(['-m', 'immediate']) + node.cleanup() + shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + + try: + self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id, options=['-j', '3']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node,tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location(self): + """ + Case: Restore instance from valid full backup to new location. + """ + node.stop(['-m', 'immediate']) + node.cleanup() + shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + + node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, fname)) + + try: + self.restore_node(backup_dir, 'node', node_new, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node_new,tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_5_jobs(self): + """ + Case: Restore instance from valid full backup to new location. + """ + node.stop(['-m', 'immediate']) + node.cleanup() + shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + + node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, fname)) + + try: + self.restore_node(backup_dir, 'node', node_new, backup_id=self.backup_id, options=['-j', '5']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(node_new,tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + node_new.cleanup() + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_new_jobs(self): + pass + +# --------------------------------------------------------- # + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_new_jobs(self): + """ + Case: Restore from backup to new location, four jobs + """ + pass + + def tearDown(self): + node.cleanup() + self.del_test_dir(module_name, fname) From a5be84af44e5eee2ccd5ddec39618df53f2add7f Mon Sep 17 00:00:00 2001 From: "s.logvinenko" Date: Mon, 2 Oct 2017 14:53:24 +0300 Subject: [PATCH 3/7] Update tests --- tests/cfs_backup_enc.py | 686 +---------------------------------- tests/cfs_backup_noenc.py | 338 ++++++++++------- tests/cfs_restore_noenc.py | 178 ++++----- tests/helpers/__init__.py | 2 +- tests/helpers/cfs_helpers.py | 53 ++- 5 files changed, 334 insertions(+), 923 deletions(-) diff --git a/tests/cfs_backup_enc.py b/tests/cfs_backup_enc.py index 412e3a63..e190a1cc 100644 --- a/tests/cfs_backup_enc.py +++ b/tests/cfs_backup_enc.py @@ -1,688 +1,14 @@ import os -import unittest -from .helpers.cfs_helpers import find_by_extensions, find_by_name -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .cfs_backup_noenc import CfsBackupNoEncTest +from .helpers.cfs_helpers import find_by_name -module_name = 'cfs_backup_noenc' +module_name = 'cfs_backup_enc' tblspace_name = 'cfs_tblspace' -class CfsBackupEncTest(ProbackupTest, unittest.TestCase): - fname = None - backup_dir = None - node = None - -# --- Begin --- # +class CfsBackupEncTest(CfsBackupNoEncTest): + # --- Begin --- # def setUp(self): - global fname - global backup_dir - global node - os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'cfs_encryption': 'on', - 'max_wal_senders': '2' - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.start() - - self.create_tblspace_in_node(node, tblspace_name, True) - - tblspace = node.safe_psql( - "postgres", - "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) - ) - self.assertTrue( - tblspace_name in tblspace and "compression=true" in tblspace, - "ERROR: The tablespace not created or it create without compressions" - ) - - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - -# --- Section: Full --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace(self): - """ - Case: Check fullbackup empty compressed tablespace - """ - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_stream(self): - """ - Case: Check fullbackup empty compressed tablespace with options stream - """ - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table(self): - """ - Case: Make full backup after created table in the tablespace - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {0}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table_stream(self): - """ - Case: Make full backup after created table in the tablespace with option --stream - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - -# --- Section: Incremental from empty tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["Status"], - "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["Status"]) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" - ) - self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - -# --- Section: Incremental from fill tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_after_create_table_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,25) i'.format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full)[0] - show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack)[0] - self.assertGreater( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"], - "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace (--stream). - Make ptrack backup after create table (--stream). - Check: incremental backup will not greater as full - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,25) i'.format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) - self.assertGreater( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"], - "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_after_create_table_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,25) i'.format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page') - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"], - "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_fullbackup_after_create_table_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace (--stream). - Make ptrack backup after create table (--stream). - Check: incremental backup will not greater as full - """ - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,256) i'.format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,25) i'.format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) - except ProbackupException as e: - self.assertTrue( - False, - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"], - "ERROR: Size of incremental backup greater as full. \n INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - -# --- end ---# - def tearDown(self): - node.cleanup() - self.del_test_dir(module_name, fname) + super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_backup_noenc.py b/tests/cfs_backup_noenc.py index 1d9ddfa8..8cd0cb7b 100644 --- a/tests/cfs_backup_noenc.py +++ b/tests/cfs_backup_noenc.py @@ -1,29 +1,22 @@ import os import unittest +import random -from .helpers.cfs_helpers import find_by_extensions, find_by_name +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_backup_enc' +module_name = 'cfs_backup_noenc' tblspace_name = 'cfs_tblspace' -class CfsBackupEncTest(ProbackupTest, unittest.TestCase): - fname = None - backup_dir = None - node = None - -# --- Begin --- # +class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): + # --- Begin --- # def setUp(self): - global fname - global backup_dir - global node + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -34,15 +27,15 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) - node.start() + self.node.start() - self.create_tblspace_in_node(node, tblspace_name, True) + self.create_tblspace_in_node(self.node, tblspace_name, True) - tblspace = node.safe_psql( + tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) ) @@ -52,11 +45,11 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) -# --- Section: Full --- # + # --- Section: Full --- # # @unittest.expectedFailure # @unittest.skip("skip") def test_fullbackup_empty_tablespace(self): @@ -66,7 +59,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -74,14 +67,14 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) @@ -94,7 +87,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -102,18 +95,18 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files found in backup dir" ) @@ -125,7 +118,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Case: Make full backup after created table in the tablespace """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -135,7 +128,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {0}".format( @@ -143,22 +136,22 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), - "ERROR: File pg_compression not found in {0}".format(os.path.join(backup_dir, 'node', backup_id)) + find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + "ERROR: File pg_compression not found in {0}".format(os.path.join(self.backup_dir, 'node', backup_id)) ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) @@ -170,7 +163,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Case: Make full backup after created table in the tablespace with option --stream """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -180,7 +173,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -188,26 +181,26 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Full backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([os.path.join(backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), + find_by_name([os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) -# --- Section: Incremental from empty tablespace --- # + # --- Section: Incremental from empty tablespace --- # # @unittest.expectedFailure # @unittest.skip("skip") def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): @@ -217,7 +210,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ try: - self.backup_node(backup_dir, 'node', node, backup_type='full') + self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -226,7 +219,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -236,7 +229,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='ptrack') except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -244,22 +237,22 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) @@ -272,7 +265,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ try: - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -281,7 +274,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -291,7 +284,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='ptrack', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -299,22 +292,22 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) @@ -327,7 +320,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ try: - self.backup_node(backup_dir, 'node', node, backup_type='full') + self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -336,7 +329,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -346,7 +339,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page') except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -354,22 +347,22 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) @@ -382,7 +375,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): """ try: - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -391,7 +384,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -401,7 +394,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id = None try: - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) + backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -409,26 +402,26 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - show_backup = self.show_pb(backup_dir, 'node', backup_id) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( "OK", show_backup["status"], "ERROR: Incremental backup status is not valid. \n Current backup status={0}".format(show_backup["status"]) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found" ) self.assertTrue( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['.cmf']), - "ERROR: .cmf files not found in backup dir" + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['.cfm']), + "ERROR: .cfm files not found in backup dir" ) self.assertFalse( - find_by_extensions([os.path.join(backup_dir, 'node', backup_id)], ['_ptrack']), + find_by_extensions([os.path.join(self.backup_dir, 'node', backup_id)], ['_ptrack']), "ERROR: _ptrack files was found in backup dir" ) -# --- Section: Incremental from fill tablespace --- # + # --- Section: Incremental from fill tablespace --- # # @unittest.expectedFailure # @unittest.skip("skip") def test_fullbackup_after_create_table_ptrack_after_create_table(self): @@ -438,7 +431,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -448,7 +441,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_full = None try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') + backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -457,7 +450,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -467,7 +460,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_ptrack = None try: - backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + backup_id_ptrack = self.backup_node(self.backup_dir, 'node', self.node, backup_type='ptrack') except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -476,8 +469,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) + show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb(self.backup_dir, 'node', backup_id_ptrack) self.assertGreater( show_backup_ptrack["data-bytes"], show_backup_full["data-bytes"], @@ -491,12 +484,12 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): """ - Case: Make full backup before created table in the tablespace (--stream). - Make ptrack backup after create table (--stream). + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). Check: incremental backup will not greater as full """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -506,7 +499,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_full = None try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -515,7 +508,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -525,7 +518,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_ptrack = None try: - backup_id_ptrack = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + backup_id_ptrack = self.backup_node(self.backup_dir, 'node', self.node, backup_type='ptrack', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -534,8 +527,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb(backup_dir, 'node', backup_id_ptrack) + show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb(self.backup_dir, 'node', backup_id_ptrack) self.assertGreater( show_backup_ptrack["data-bytes"], show_backup_full["data-bytes"], @@ -554,7 +547,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): Check: incremental backup will not greater as full """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -564,7 +557,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_full = None try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full') + backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -573,7 +566,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -583,7 +576,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_page = None try: - backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_page = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page') except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -592,8 +585,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) + show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb(self.backup_dir, 'node', backup_id_page) self.assertGreater( show_backup_page["data-bytes"], show_backup_full["data-bytes"], @@ -607,12 +600,12 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_fullbackup_after_create_table_page_after_create_table_stream(self): """ - Case: Make full backup before created table in the tablespace (--stream). - Make ptrack backup after create table (--stream). + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). Check: incremental backup will not greater as full """ - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -622,7 +615,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_full = None try: - backup_id_full = self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + backup_id_full = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Full backup failed.\n {0} \n {1}".format( @@ -631,7 +624,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - node.safe_psql( + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ @@ -641,7 +634,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): backup_id_page = None try: - backup_id_page = self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream']) + backup_id_page = self.backup_node(self.backup_dir, 'node', self.node, backup_type='page', options=['--stream']) except ProbackupException as e: self.fail( "ERROR: Incremental backup failed.\n {0} \n {1}".format( @@ -650,8 +643,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) - show_backup_full = self.show_pb(backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb(backup_dir, 'node', backup_id_page) + show_backup_full = self.show_pb(self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb(self.backup_dir, 'node', backup_id_page) self.assertGreater( show_backup_page["data-bytes"], show_backup_full["data-bytes"], @@ -661,43 +654,130 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): ) ) -# --- Make backup with not valid data (broken .cfm) --- # + # --- Make backup with not valid data(broken .cfm) --- # # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_delete_random_cfm_file_from_tablespace_dir(self): - pass + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions([self.get_tblspace_path(self.node,tblspace_name)],['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + os.remove(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) # @unittest.expectedFailure # @unittest.skip("skip") def test_delete_file_pg_compression_from_tablespace_dir(self): - os.remove(find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression'])[0]) + os.remove(find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression'])[0]) self.assertRaises( ProbackupException, - self.backup_node,backup_dir, 'node', node, backup_type='full' + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_delete_random_data_file_from_tablespace_dir(self): - pass + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern([self.get_tblspace_path(self.node,tblspace_name)],'^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + os.remove(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_broken_random_cfm_file_into_tablespace_dir(self): - pass + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions([self.get_tblspace_path(self.node,tblspace_name)],['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + corrupt_file(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_broken_random_data_file_into_tablespace_dir(self): - pass + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,256) i'.format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern([self.get_tblspace_path(self.node,tblspace_name)],'^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + corrupt_file(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_broken_file_pg_compression_into_tablespace_dir(self): - pass -# --- Validation backup --- # + corrupted_file = find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression'])[0] + + self.assertTrue( + corrupt_file(corrupted_file), + "ERROR: File is not corrupted or it missing" + ) + + self.assertRaises( + ProbackupException, + self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' + ) + + # --- Validation backup --- # # @unittest.expectedFailure @unittest.skip("skip") def test_delete_random_cfm_file_from_backup_dir(self): @@ -706,8 +786,8 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_delete_file_pg_compression_from_backup_dir(self): - self.backup_node(backup_dir, 'node', node, backup_type = 'full') - show_backup = self.show_pb(backup_dir,'node')[0] + self.backup_node(self.backup_dir, 'node', self.node, backup_type = 'full') + show_backup = self.show_pb(self.backup_dir,'node')[0] self.assertEqual( "OK", @@ -715,9 +795,9 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): "ERROR: Backup is not valid. \n Backup status: %s" % show_backup["Status"] ) - os.remove(find_by_name([os.path.join(backup_dir, 'backups', 'node', show_backup["ID"])], ['pg_compression'])[0]) + os.remove(find_by_name([os.path.join(self.backup_dir, 'backups', 'node', show_backup["ID"])], ['pg_compression'])[0]) - self.assertRaises(ProbackupException, self.validate_pb, backup_dir, 'node') + self.assertRaises(ProbackupException, self.validate_pb, self.backup_dir, 'node') # @unittest.expectedFailure @unittest.skip("skip") @@ -739,7 +819,7 @@ class CfsBackupEncTest(ProbackupTest, unittest.TestCase): def test_broken_file_pg_compression_into_backup_dir(self): pass -# --- end ---# + # --- End ---# def tearDown(self): - node.cleanup() - self.del_test_dir(module_name, fname) + self.node.cleanup() + self.del_test_dir(module_name, self.fname) diff --git a/tests/cfs_restore_noenc.py b/tests/cfs_restore_noenc.py index 1e698f57..0af9f7b7 100644 --- a/tests/cfs_restore_noenc.py +++ b/tests/cfs_restore_noenc.py @@ -22,21 +22,13 @@ tblspace_name = 'cfs_tblspace_noenc' tblspace_name_new = 'cfs_tblspace_new' -class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): - fname = None - backup_dir = None - node = None - +class CfsRestoreBaseTest(ProbackupTest, unittest.TestCase): def setUp(self): - global fname - global backup_dir - global node + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -47,16 +39,18 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) - node.start() + self.node.start() - self.create_tblspace_in_node(node, tblspace_name, True) + self.add_data_in_cluster() + + self.create_tblspace_in_node(self.node, tblspace_name, True) self.backup_id = None try: - self.backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') except ProbackupException as e: self.fail( "ERROR: Full backup failed \n {0} \n {1}".format( @@ -65,18 +59,27 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): ) ) + def add_data_in_cluster(self): + pass + + def tearDown(self): + self.node.cleanup() + self.del_test_dir(module_name, self.fname) + + +class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBaseTest): # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_empty_tablespace_from_fullbackup(self): """ Case: Restore empty tablespace from valid full backup. """ - node.stop({"-m": "immediate"}) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop({"-m": "immediate"}) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) try: - self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id) + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) except ProbackupException as e: self.fail( "ERROR: Restore failed. \n {0} \n {1}".format( @@ -85,12 +88,12 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ["pg_compression"]), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." ) try: - node.start() + self.node.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -98,7 +101,7 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): repr(e.message) ) ) - tblspace = node.safe_psql( + tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) ) @@ -107,79 +110,34 @@ class CfsRestoreNoencEmptyTablespaceTest(ProbackupTest, unittest.TestCase): "ERROR: The tablespace not restored or it restored without compressions" ) - def tearDown(self): - node.cleanup() - self.del_test_dir(module_name, fname) - -class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): - fname = None - backup_dir = None - node = None - - def setUp(self): - global fname - global backup_dir - global node - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'cfs_encryption': 'off', - 'max_wal_senders': '2' - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.start() - - self.create_tblspace_in_node(node, tblspace_name, True) - - node.safe_psql( +class CfsRestoreNoencTest(CfsRestoreBaseTest): + def add_data_in_cluster(self): + self.node.safe_psql( "postgres", 'CREATE TABLE {0} TABLESPACE {1} \ AS SELECT i AS id, MD5(i::text) AS text, \ MD5(repeat(i::text,10))::tsvector AS tsvector \ FROM generate_series(0,1e5) i'.format('t1', tblspace_name) ) - self.table_t1 = node.safe_psql( + self.table_t1 = self.node.safe_psql( "postgres", "SELECT * FROM t1" ) - self.backup_id = None - try: - self.backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) -# --- Restore from full backup ---# + # --- Restore from full backup ---# # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_from_fullbackup_to_old_location(self): """ Case: Restore instance from valid full backup to old location. """ - node.stop(['-m', 'immediate']) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) try: - self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id) + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -188,11 +146,11 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node, tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) try: - node.start() + self.node.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -202,7 +160,7 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) self.assertEqual( - repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), repr(self.table_t1) ) @@ -212,12 +170,12 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): """ Case: Restore instance from valid full backup to old location. """ - node.stop(['-m', 'immediate']) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) try: - self.restore_node(backup_dir, 'node', node, backup_id=self.backup_id, options=['-j', '3']) + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -226,11 +184,11 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node,tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) try: - node.start() + self.node.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -240,7 +198,7 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) self.assertEqual( - repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), repr(self.table_t1) ) @@ -250,14 +208,14 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): """ Case: Restore instance from valid full backup to new location. """ - node.stop(['-m', 'immediate']) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, fname)) + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) try: - self.restore_node(backup_dir, 'node', node_new, backup_id=self.backup_id) + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -266,11 +224,11 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node_new,tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node_new, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) try: - node_new.start() + self.node_new.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -280,10 +238,10 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) self.assertEqual( - repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), repr(self.table_t1) ) - node_new.cleanup() + self.node_new.cleanup() # @unittest.expectedFailure # @unittest.skip("skip") @@ -291,14 +249,14 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): """ Case: Restore instance from valid full backup to new location. """ - node.stop(['-m', 'immediate']) - node.cleanup() - shutil.rmtree(self.get_tblspace_path(node, tblspace_name)) + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, fname)) + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) try: - self.restore_node(backup_dir, 'node', node_new, backup_id=self.backup_id, options=['-j', '5']) + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id, options=['-j', '5']) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -307,11 +265,11 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) ) self.assertTrue( - find_by_name([self.get_tblspace_path(node_new,tblspace_name)], ['pg_compression']), + find_by_name([self.get_tblspace_path(self.node_new, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in backup dir" ) try: - node_new.start() + self.node_new.start() except ProbackupException as e: self.fail( "ERROR: Instance not started after restore. \n {0} \n {1}".format( @@ -321,10 +279,10 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): ) self.assertEqual( - repr(node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), repr(self.table_t1) ) - node_new.cleanup() + self.node_new.cleanup() # @unittest.expectedFailure @unittest.skip("skip") @@ -391,7 +349,3 @@ class CfsRestoreNoencTest(ProbackupTest, unittest.TestCase): Case: Restore from backup to new location, four jobs """ pass - - def tearDown(self): - node.cleanup() - self.del_test_dir(module_name, fname) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index 769446ad..ac64c423 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['ptrack_helpers', 'expected_errors'] +__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] #from . import * \ No newline at end of file diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py index d35050f1..67e2b331 100644 --- a/tests/helpers/cfs_helpers.py +++ b/tests/helpers/cfs_helpers.py @@ -1,4 +1,8 @@ import os +import re +import random +import string + def find_by_extensions(dirs=None, extensions=None): """ @@ -22,6 +26,30 @@ def find_by_extensions(dirs=None, extensions=None): return files + +def find_by_pattern(dirs=None, pattern=None): + """ + find_by_pattern(['path1','path2'],'^.*/*.txt') + :return: + Return list of files include full path by pattern + """ + files = [] + new_dirs = [] + + if dirs is not None and pattern is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if re.match(pattern,d): + files.append(d) + + if new_dirs: + files.extend(find_by_pattern(new_dirs, pattern)) + + return files + + def find_by_name(dirs=None, filename=None): files = [] new_dirs = [] @@ -37,4 +65,27 @@ def find_by_name(dirs=None, filename=None): if new_dirs: files.extend(find_by_name(new_dirs, filename)) - return files \ No newline at end of file + return files + + +def corrupt_file(filename): + file_size = None + try: + file_size = os.path.getsize(filename) + except OSError: + return False + + try: + with open(filename, "rb+") as f: + f.seek(random.randint(int(0.1*file_size),int(0.8*file_size))) + f.write(random_string(0.1*file_size)) + f.close() + except OSError: + return False + + return True + + +def random_string(n): + a = string.ascii_letters + string.digits + return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file From 948927142b889d6da0ff9532dc27eefb905c52bb Mon Sep 17 00:00:00 2001 From: "s.logvinenko" Date: Mon, 2 Oct 2017 15:33:34 +0300 Subject: [PATCH 4/7] Update tests --- tests/cfs_restore_noenc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cfs_restore_noenc.py b/tests/cfs_restore_noenc.py index 0af9f7b7..ca603c6f 100644 --- a/tests/cfs_restore_noenc.py +++ b/tests/cfs_restore_noenc.py @@ -44,10 +44,10 @@ class CfsRestoreBaseTest(ProbackupTest, unittest.TestCase): self.set_archiving(self.backup_dir, 'node', self.node) self.node.start() + self.create_tblspace_in_node(self.node, tblspace_name, True) self.add_data_in_cluster() - self.create_tblspace_in_node(self.node, tblspace_name, True) self.backup_id = None try: self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') From dd14c27f38a9fb2e0782304bc3b632a765307d21 Mon Sep 17 00:00:00 2001 From: "s.logvinenko" Date: Mon, 2 Oct 2017 17:35:20 +0300 Subject: [PATCH 5/7] Update tests. Add cfs_validate_backup test --- tests/__init__.py | 6 +- tests/cfs_backup_enc.py | 1 - tests/cfs_restore_enc.py | 19 +++++++ tests/cfs_restore_noenc.py | 104 ++++++++++++++++++++++++++++++++--- tests/cfs_validate_backup.py | 25 +++++++++ 5 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 tests/cfs_restore_enc.py create mode 100644 tests/cfs_validate_backup.py diff --git a/tests/__init__.py b/tests/__init__.py index 21ee595a..44b620d1 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,7 +7,8 @@ from . import init_test, option_test, show_test, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ - cfs_backup_noenc, cfs_backup_enc, cfs_restore_noenc + cfs_backup_noenc, cfs_backup_enc, cfs_restore_noenc, cfs_restore_enc, \ + cfs_validate_backup def load_tests(loader, tests, pattern): @@ -40,7 +41,8 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(cfs_backup_noenc)) suite.addTests(loader.loadTestsFromModule(cfs_backup_enc)) suite.addTests(loader.loadTestsFromModule(cfs_restore_noenc)) - + suite.addTests(loader.loadTestsFromModule(cfs_restore_enc)) + suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) return suite # ToDo: diff --git a/tests/cfs_backup_enc.py b/tests/cfs_backup_enc.py index e190a1cc..51171ff8 100644 --- a/tests/cfs_backup_enc.py +++ b/tests/cfs_backup_enc.py @@ -1,7 +1,6 @@ import os from .cfs_backup_noenc import CfsBackupNoEncTest -from .helpers.cfs_helpers import find_by_name module_name = 'cfs_backup_enc' tblspace_name = 'cfs_tblspace' diff --git a/tests/cfs_restore_enc.py b/tests/cfs_restore_enc.py new file mode 100644 index 00000000..32d88489 --- /dev/null +++ b/tests/cfs_restore_enc.py @@ -0,0 +1,19 @@ +import os + +from .cfs_restore_noenc import CfsRestoreNoencEmptyTablespaceTest,CfsRestoreNoencTest + +module_name = 'cfs_restore_enc' + + +class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): + # --- Begin --- # + def setUp(self): + os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" + super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() + + +class CfsRestoreEncTest(CfsRestoreNoencTest): + # --- Begin --- # + def setUp(self): + os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" + super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_restore_noenc.py b/tests/cfs_restore_noenc.py index ca603c6f..4451f040 100644 --- a/tests/cfs_restore_noenc.py +++ b/tests/cfs_restore_noenc.py @@ -12,7 +12,7 @@ import os import unittest import shutil -from .helpers.cfs_helpers import find_by_extensions, find_by_name +from .helpers.cfs_helpers import find_by_name from .helpers.ptrack_helpers import ProbackupTest, ProbackupException @@ -22,7 +22,7 @@ tblspace_name = 'cfs_tblspace_noenc' tblspace_name_new = 'cfs_tblspace_new' -class CfsRestoreBaseTest(ProbackupTest, unittest.TestCase): +class CfsRestoreBase(ProbackupTest, unittest.TestCase): def setUp(self): self.fname = self.id().split('.')[3] self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') @@ -67,7 +67,7 @@ class CfsRestoreBaseTest(ProbackupTest, unittest.TestCase): self.del_test_dir(module_name, self.fname) -class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBaseTest): +class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_empty_tablespace_from_fullbackup(self): @@ -111,7 +111,7 @@ class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBaseTest): ) -class CfsRestoreNoencTest(CfsRestoreBaseTest): +class CfsRestoreNoencTest(CfsRestoreBase): def add_data_in_cluster(self): self.node.safe_psql( "postgres", @@ -147,7 +147,7 @@ class CfsRestoreNoencTest(CfsRestoreBaseTest): ) self.assertTrue( find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" + "ERROR: File pg_compression not found in tablespace dir" ) try: self.node.start() @@ -285,14 +285,100 @@ class CfsRestoreNoencTest(CfsRestoreBaseTest): self.node_new.cleanup() # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): - pass + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=[ + "-T %s = %s".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): - pass + self.node.stop(['-m', 'immediate']) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=[ + "j", + "3", + "-T %s = %s".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) # @unittest.expectedFailure @unittest.skip("skip") diff --git a/tests/cfs_validate_backup.py b/tests/cfs_validate_backup.py new file mode 100644 index 00000000..1da24fd7 --- /dev/null +++ b/tests/cfs_validate_backup.py @@ -0,0 +1,25 @@ +import os +import unittest +import random + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_validate_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): + def setUp(self): + pass + + def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): + pass + + def tearDown(self): + pass + + +class CfsValidateBackupNoenc(CfsValidateBackupNoenc): + os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" + super(CfsValidateBackupNoenc).setUp() \ No newline at end of file From 7bfa61ad7e36d60e2249233449b99f09dba4c708 Mon Sep 17 00:00:00 2001 From: sfalkon Date: Tue, 3 Oct 2017 14:59:30 +0300 Subject: [PATCH 6/7] Update cfs-tests --- tests/__init__.py | 10 ++-- tests/{cfs_backup_noenc.py => cfs_backup.py} | 51 +++---------------- tests/cfs_backup_enc.py | 13 ----- .../{cfs_restore_noenc.py => cfs_restore.py} | 21 ++++++-- tests/cfs_restore_enc.py | 19 ------- 5 files changed, 29 insertions(+), 85 deletions(-) rename tests/{cfs_backup_noenc.py => cfs_backup.py} (95%) delete mode 100644 tests/cfs_backup_enc.py rename tests/{cfs_restore_noenc.py => cfs_restore.py} (96%) delete mode 100644 tests/cfs_restore_enc.py diff --git a/tests/__init__.py b/tests/__init__.py index 44b620d1..cc59dc60 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,8 +7,7 @@ from . import init_test, option_test, show_test, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ - cfs_backup_noenc, cfs_backup_enc, cfs_restore_noenc, cfs_restore_enc, \ - cfs_validate_backup + cfs_backup, cfs_restore def load_tests(loader, tests, pattern): @@ -38,11 +37,8 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(compression)) suite.addTests(loader.loadTestsFromModule(page)) suite.addTests(loader.loadTestsFromModule(archive)) - suite.addTests(loader.loadTestsFromModule(cfs_backup_noenc)) - suite.addTests(loader.loadTestsFromModule(cfs_backup_enc)) - suite.addTests(loader.loadTestsFromModule(cfs_restore_noenc)) - suite.addTests(loader.loadTestsFromModule(cfs_restore_enc)) - suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) + suite.addTests(loader.loadTestsFromModule(cfs_backup)) + suite.addTests(loader.loadTestsFromModule(cfs_restore)) return suite # ToDo: diff --git a/tests/cfs_backup_noenc.py b/tests/cfs_backup.py similarity index 95% rename from tests/cfs_backup_noenc.py rename to tests/cfs_backup.py index 8cd0cb7b..62128600 100644 --- a/tests/cfs_backup_noenc.py +++ b/tests/cfs_backup.py @@ -5,7 +5,7 @@ import random from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_backup_noenc' +module_name = 'cfs_backup' tblspace_name = 'cfs_tblspace' @@ -777,49 +777,14 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): self.backup_node,self.backup_dir, 'node', self.node, backup_type='full' ) - # --- Validation backup --- # - # @unittest.expectedFailure - @unittest.skip("skip") - def test_delete_random_cfm_file_from_backup_dir(self): - pass - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_delete_file_pg_compression_from_backup_dir(self): - self.backup_node(self.backup_dir, 'node', self.node, backup_type = 'full') - show_backup = self.show_pb(self.backup_dir,'node')[0] - - self.assertEqual( - "OK", - show_backup["Status"], - "ERROR: Backup is not valid. \n Backup status: %s" % show_backup["Status"] - ) - - os.remove(find_by_name([os.path.join(self.backup_dir, 'backups', 'node', show_backup["ID"])], ['pg_compression'])[0]) - - self.assertRaises(ProbackupException, self.validate_pb, self.backup_dir, 'node') - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_delete_random_data_file_from_backup_dir(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_broken_random_cfm_file_into_backup_dir(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_broken_random_data_file_into_backup_dir(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_broken_file_pg_compression_into_backup_dir(self): - pass - # --- End ---# def tearDown(self): self.node.cleanup() self.del_test_dir(module_name, self.fname) + + +class CfsBackupEncTest(CfsBackupNoEncTest): + # --- Begin --- # + def setUp(self): + os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" + super(CfsBackupEncTest, self).setUp() \ No newline at end of file diff --git a/tests/cfs_backup_enc.py b/tests/cfs_backup_enc.py deleted file mode 100644 index 51171ff8..00000000 --- a/tests/cfs_backup_enc.py +++ /dev/null @@ -1,13 +0,0 @@ -import os - -from .cfs_backup_noenc import CfsBackupNoEncTest - -module_name = 'cfs_backup_enc' -tblspace_name = 'cfs_tblspace' - - -class CfsBackupEncTest(CfsBackupNoEncTest): - # --- Begin --- # - def setUp(self): - os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" - super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_restore_noenc.py b/tests/cfs_restore.py similarity index 96% rename from tests/cfs_restore_noenc.py rename to tests/cfs_restore.py index 4451f040..4a802ae2 100644 --- a/tests/cfs_restore_noenc.py +++ b/tests/cfs_restore.py @@ -16,9 +16,9 @@ from .helpers.cfs_helpers import find_by_name from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_restore_noenc' +module_name = 'cfs_restore' -tblspace_name = 'cfs_tblspace_noenc' +tblspace_name = 'cfs_tblspace' tblspace_name_new = 'cfs_tblspace_new' @@ -132,7 +132,7 @@ class CfsRestoreNoencTest(CfsRestoreBase): """ Case: Restore instance from valid full backup to old location. """ - self.node.stop(['-m', 'immediate']) + self.node.stop() self.node.cleanup() shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) @@ -145,6 +145,7 @@ class CfsRestoreNoencTest(CfsRestoreBase): repr(e.message) ) ) + self.assertTrue( find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), "ERROR: File pg_compression not found in tablespace dir" @@ -435,3 +436,17 @@ class CfsRestoreNoencTest(CfsRestoreBase): Case: Restore from backup to new location, four jobs """ pass + + +class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): + # --- Begin --- # + def setUp(self): + os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" + super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() + + +class CfsRestoreEncTest(CfsRestoreNoencTest): + # --- Begin --- # + def setUp(self): + os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" + super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_restore_enc.py b/tests/cfs_restore_enc.py deleted file mode 100644 index 32d88489..00000000 --- a/tests/cfs_restore_enc.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -from .cfs_restore_noenc import CfsRestoreNoencEmptyTablespaceTest,CfsRestoreNoencTest - -module_name = 'cfs_restore_enc' - - -class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): - # --- Begin --- # - def setUp(self): - os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" - super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() - - -class CfsRestoreEncTest(CfsRestoreNoencTest): - # --- Begin --- # - def setUp(self): - os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" - super(CfsRestoreNoencTest, self).setUp() From 2593e10a9ba9870f44abd5c5fdd9e5d664b0c187 Mon Sep 17 00:00:00 2001 From: "s.logvinenko" Date: Wed, 4 Oct 2017 17:03:03 +0300 Subject: [PATCH 7/7] Merge branch 'master' of https://git.postgrespro.ru/pgpro-dev/pg_probackup into cfs Conflicts: tests/__init__.py --- tests/helpers/ptrack_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 677fc6d5..ea93d9b2 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -4,7 +4,7 @@ from sys import exit, argv, version_info import subprocess import shutil import six -from testgres import get_new_node, clean_all +import testgres import hashlib import re import pwd @@ -153,7 +153,7 @@ class ProbackupTest(object): real_base_dir = os.path.join(self.tmp_path, base_dir) shutil.rmtree(real_base_dir, ignore_errors=True) - node = get_new_node('test', base_dir=real_base_dir) + node = testgres.get_new_node('test', base_dir=real_base_dir) node.init(initdb_params=initdb_params) # Sane default parameters, not a shit with fsync = off from testgres @@ -585,7 +585,7 @@ class ProbackupTest(object): def del_test_dir(self, module_name, fname): """ Del testdir and optimistically try to del module dir""" try: - clean_all() + testgres.clean_all() except: pass