From ab1965d16d4f10d94b37a082517f6c7e15009710 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 11 May 2017 12:35:51 +0300 Subject: [PATCH 01/37] Version 1.1.11 --- pg_probackup.c | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_probackup.c b/pg_probackup.c index 24a0d547..13d329c3 100644 --- a/pg_probackup.c +++ b/pg_probackup.c @@ -16,7 +16,7 @@ #include #include -const char *PROGRAM_VERSION = "1.1.10"; +const char *PROGRAM_VERSION = "1.1.11"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 41877ab8..adc3ad0d 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 1.1.5 +pg_probackup 1.1.11 From ec8d9b8ce4c8528b594e1b57ebfcefbe943f776d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 24 May 2017 16:43:16 +0300 Subject: [PATCH 02/37] ptrack page header fix --- tests/backup_test.py | 14 +-- tests/expected/option_help.out | 10 +- tests/expected/option_version.out | 2 +- tests/ptrack_clean.py | 10 +- tests/ptrack_cluster.py | 25 ++--- tests/ptrack_helpers.py | 149 ++++++++++++------------- tests/ptrack_move_to_tablespace.py | 3 +- tests/ptrack_recovery.py | 3 +- tests/ptrack_vacuum.py | 8 +- tests/ptrack_vacuum_bits_frozen.py | 3 +- tests/ptrack_vacuum_bits_visibility.py | 3 +- tests/ptrack_vacuum_full.py | 3 +- tests/ptrack_vacuum_truncate.py | 3 +- tests/restore_test.py | 13 --- tests/retention_test.py | 2 - tests/validate_test.py | 49 +++----- 16 files changed, 125 insertions(+), 175 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 6c654d49..62c5ceff 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -17,7 +17,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -74,7 +73,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): def test_smooth_checkpoint(self): """full backup with smooth checkpoint""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -94,7 +92,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): def test_page_backup_without_full(self): """page-level backup without validated full backup""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -114,10 +111,12 @@ class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("123") def test_ptrack_threads(self): """ptrack multi thread backup mode""" - node = self.make_bnode( - base_dir="tmp_dirs/backup/ptrack_threads_4", - options={"ptrack_enable": "on", 'max_wal_senders': '2'} - ) + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), + set_archiving=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} + ) node.start() self.assertEqual(self.init_pb(node), six.b("")) @@ -137,7 +136,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): def test_ptrack_threads_stream(self): """ptrack multi thread backup mode and stream""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums'], diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index d2a4957c..53b46eb4 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -1,7 +1,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. - pg_probackup help + pg_probackup help [COMMAND] pg_probackup version @@ -21,20 +21,20 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-d dbname] [-h host] [-p port] [-U username] pg_probackup restore -B backup-dir - [-D pgdata-dir] [-i backup-id] + [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v] [--time=time|--xid=xid [--inclusive=boolean]] [--timeline=timeline] [-T OLDDIR=NEWDIR] pg_probackup validate -B backup-dir - [-D pgdata-dir] [-i backup-id] + [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v] [--time=time|--xid=xid [--inclusive=boolean]] - [--timeline=timeline] [-T OLDDIR=NEWDIR] + [--timeline=timeline] pg_probackup show -B backup-dir [-i backup-id] pg_probackup delete -B backup-dir - [--wal] [-i backup-id | --expired] [--force] + [--wal] [-i backup-id | --expired] Read the website for details. Report bugs to . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 5c184bf6..adc3ad0d 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 1.1.11 \ No newline at end of file +pg_probackup 1.1.11 diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 2a458c75..597ea7e7 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -9,10 +9,10 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) def teardown(self): - # clean_all() stop_all() -# @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_clean(self): fname = self.id().split('.')[3] node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), @@ -45,7 +45,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'] = self.get_fork_path(node, i) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) # Update everything, vacuum it and make PTRACK BACKUP @@ -62,7 +62,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'] = self.get_fork_path(node, i) # # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) @@ -81,7 +81,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'] = self.get_fork_path(node, i) # # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index 8a902fc3..e4525bfd 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -12,10 +12,9 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # clean_all() stop_all() -# @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_btree(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -63,7 +62,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) @@ -71,10 +70,9 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.clean_pb(node) node.stop() - @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_spgist(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -122,7 +120,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) @@ -130,10 +128,9 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.clean_pb(node) node.stop() - @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_brin(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -181,7 +178,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) @@ -189,10 +186,9 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.clean_pb(node) node.stop() - @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_gist(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -240,7 +236,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) @@ -248,10 +244,9 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.clean_pb(node) node.stop() - @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_gin(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -299,7 +294,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_helpers.py b/tests/ptrack_helpers.py index b233e4d5..40c77c1a 100644 --- a/tests/ptrack_helpers.py +++ b/tests/ptrack_helpers.py @@ -152,29 +152,6 @@ class ProbackupTest(object): def backup_dir(self, node): return os.path.abspath("%s/backup" % node.base_dir) - def make_bnode(self, base_dir=None, allows_streaming=False, options={}): - real_base_dir = os.path.join(self.dir_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - - node = get_new_node('test', base_dir=real_base_dir) - node.init(allows_streaming=allows_streaming) - - if not allows_streaming: - node.append_conf("postgresql.auto.conf", "wal_level = hot_standby") - node.append_conf("postgresql.auto.conf", "archive_mode = on") - node.append_conf( - "postgresql.auto.conf", - """archive_command = 'cp "%%p" "%s/%%f"'""" % os.path.abspath(self.arcwal_dir(node)) - ) - - for key, value in six.iteritems(options): - node.append_conf("postgresql.conf", "%s = %s" % (key, value)) - - return node - -# def print_started(self, fname): -# print - def make_simple_node(self, base_dir=None, set_replication=False, set_archiving=False, initdb_params=[], pg_options={}): real_base_dir = os.path.join(self.dir_path, base_dir) @@ -184,6 +161,7 @@ class ProbackupTest(object): node.init(initdb_params=initdb_params) # Sane default parameters, not a shit with fsync = off from testgres + node.append_conf("postgresql.auto.conf", "{0} = {1}".format('shared_buffers', '10MB')) node.append_conf("postgresql.auto.conf", "{0} = {1}".format('fsync', 'on')) node.append_conf("postgresql.auto.conf", "{0} = {1}".format('wal_level', 'minimal')) @@ -199,7 +177,6 @@ class ProbackupTest(object): self.set_archiving_conf(node, self.arcwal_dir(node)) return node - def create_tblspace_in_node(self, node, tblspc_name, cfs=False): res = node.execute( "postgres", "select exists (select 1 from pg_tablespace where spcname = '{0}')".format( @@ -236,12 +213,16 @@ class ProbackupTest(object): os.close(file) return md5_per_page - def get_ptrack_bits_per_page_for_fork(self, file, size): + def get_ptrack_bits_per_page_for_fork(self, node, file, size): + if self.get_pgpro_edition(node) == 'enterprise': + header_size = 48 + else: + header_size = 24 ptrack_bits_for_fork = [] byte_size = os.path.getsize(file + '_ptrack') - byte_size_minus_header = byte_size - 24 + byte_size_minus_header = byte_size - header_size file = os.open(file + '_ptrack', os.O_RDONLY) - os.lseek(file, 24, 0) + os.lseek(file, header_size, 0) lot_of_bytes = os.read(file, byte_size_minus_header) for byte in lot_of_bytes: byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1] @@ -316,20 +297,32 @@ class ProbackupTest(object): success = False self.assertEqual(success, True) - def run_pb(self, command): + def run_pb(self, command, async=False): try: -# print [self.probackup_path] + command - output = subprocess.check_output( - [self.probackup_path] + command, - stderr=subprocess.STDOUT, - env=self.test_env - ) + #print ' '.join(map(str,[self.probackup_path] + command)) + if async is True: + return subprocess.Popen( + [self.probackup_path] + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ) if command[0] == 'backup': if '-q' in command or '--quiet' in command: return None + elif '-v' in command or '--verbose' in command: + return output else: # return backup ID - return output.split()[2] + for line in output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] else: return output except subprocess.CalledProcessError as e: @@ -346,50 +339,38 @@ class ProbackupTest(object): def clean_pb(self, node): shutil.rmtree(self.backup_dir(node), ignore_errors=True) - def backup_pb(self, node, backup_type="full", options=[]): + def backup_pb(self, node=None, data_dir=None, backup_dir=None, backup_type="full", options=[], async=False): + if data_dir is None: + data_dir = node.data_dir + if backup_dir is None: + backup_dir = self.backup_dir(node) + cmd_list = [ "backup", - "-D", node.data_dir, - "-B", self.backup_dir(node), + "-B", backup_dir, + "-D", data_dir, "-p", "%i" % node.port, "-d", "postgres" ] if backup_type: cmd_list += ["-b", backup_type] - return self.run_pb(cmd_list + options) + return self.run_pb(cmd_list + options, async) - def backup_pb_proc(self, node, backup_type="full", - stdout=None, stderr=None, options=[]): - cmd_list = [ - self.probackup_path, - "backup", - "-D", node.data_dir, - "-B", self.backup_dir(node), - "-p", "%i" % (node.port), - "-d", "postgres" - ] - if backup_type: - cmd_list += ["-b", backup_type] + def restore_pb(self, node=None, backup_dir=None, data_dir=None, id=None, options=[]): + if data_dir is None: + data_dir = node.data_dir + if backup_dir is None: + backup_dir = self.backup_dir(node) - proc = subprocess.Popen( - cmd_list + options, - stdout=stdout, - stderr=stderr - ) - - return proc - - def restore_pb(self, node, id=None, options=[]): cmd_list = [ "restore", - "-D", node.data_dir, - "-B", self.backup_dir(node) + "-B", backup_dir, + "-D", data_dir ] if id: cmd_list += ["-i", id] - # print(cmd_list) return self.run_pb(cmd_list + options) def show_pb(self, node, id=None, options=[], as_text=False): @@ -417,13 +398,17 @@ class ProbackupTest(object): body = body[::-1] # split string in list with string for every header element header_split = re.split(" +", header) - # CRUNCH, remove last item, because it empty, like that '' - header_split.pop() + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) for backup_record in body: # split string in list with string for every backup record element backup_record_split = re.split(" +", backup_record) - # CRUNCH, remove last item, because it empty, like that '' - backup_record_split.pop() + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) if len(header_split) != len(backup_record_split): print warning.format( header=header, body=body, @@ -500,25 +485,34 @@ class ProbackupTest(object): out_dict[key.strip()] = value.strip(" '").replace("'\n", "") return out_dict - def set_archiving_conf(self, node, archive_dir): + def set_archiving_conf(self, node, archive_dir=False, replica=False): + if not archive_dir: + archive_dir = self.arcwal_dir(node) + + if replica: + archive_mode = 'always' + node.append_conf('postgresql.auto.conf', 'hot_standby = on') + else: + archive_mode = 'on' + node.append_conf( "postgresql.auto.conf", "wal_level = archive" ) node.append_conf( "postgresql.auto.conf", - "archive_mode = on" + "archive_mode = {0}".format(archive_mode) ) if os.name == 'posix': node.append_conf( "postgresql.auto.conf", "archive_command = 'test ! -f {0}/%f && cp %p {0}/%f'".format(archive_dir) ) - elif os.name == 'nt': - node.append_conf( - "postgresql.auto.conf", - "archive_command = 'copy %p {0}\\%f'".format(archive_dir) - ) + #elif os.name == 'nt': + # node.append_conf( + # "postgresql.auto.conf", + # "archive_command = 'copy %p {0}\\%f'".format(archive_dir) + # ) def wrong_wal_clean(self, node, wal_size): wals_dir = os.path.join(self.backup_dir(node), "wal") @@ -536,4 +530,9 @@ class ProbackupTest(object): var = node.execute("postgres", "select setting from pg_settings where name = 'wal_block_size'") return int(var[0][0]) -# def ptrack_node(self, ptrack_enable=False, wal_level='minimal', max_wal_senders='2', allow_replication=True) + def get_pgpro_edition(self, node): + if node.execute("postgres", "select exists(select 1 from pg_proc where proname = 'pgpro_edition')")[0][0]: + var = node.execute("postgres", "select pgpro_edition()") + return str(var[0][0]) + else: + return False diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index 5acd5bbd..d43282f1 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -17,7 +17,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_recovery(self): fname = self.id().split(".")[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -49,7 +48,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'] = self.get_fork_path(node, i) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index d2a78bd9..73e9e085 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -17,7 +17,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_recovery(self): fname = self.id().split(".")[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -51,7 +50,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): for i in idx_ptrack: # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index 6cb66b9a..484c5c50 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -12,10 +12,10 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # clean_all() stop_all() -# @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -51,7 +51,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) # Delete some rows, vacuum it and make checkpoint @@ -69,7 +69,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index ba7e0a39..75d25909 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -14,7 +14,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): stop_all() def test_ptrack_vacuum_bits_frozen(self): - print 'test_ptrack_vacuum_bits_frozen started' node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_bits_frozen", set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -60,7 +59,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py index f24ae240..4fc12419 100644 --- a/tests/ptrack_vacuum_bits_visibility.py +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -14,7 +14,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): stop_all() def test_ptrack_vacuum_bits_visibility(self): - print 'test_ptrack_vacuum_bits_visibility started' node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_bits_visibility", set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -60,7 +59,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index ade1fd30..98af70be 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -27,7 +27,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_full(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -75,7 +74,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity, the most important part self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 035c7e46..eba15da4 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -14,7 +14,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): stop_all() def test_ptrack_vacuum_truncate(self): - print 'test_ptrack_vacuum_truncate started' node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_truncate", set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -62,7 +61,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/restore_test.py b/tests/restore_test.py index 8ef45527..ea35a0b1 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -22,7 +22,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_full_to_latest(self): """recovery to latest from full backup""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -61,7 +60,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_full_page_to_latest(self): """recovery to latest from full + page backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -101,7 +99,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_to_timeline(self): """recovery to target timeline""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -156,7 +153,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_to_time(self): """recovery to target timeline""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -195,7 +191,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_to_xid(self): """recovery to target xid""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -249,7 +244,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_full_ptrack(self): """recovery to latest from full + ptrack backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -297,7 +291,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_full_ptrack_ptrack(self): """recovery to latest from full + ptrack + ptrack backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -352,7 +345,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums'], @@ -397,7 +389,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_full_ptrack_under_load(self): """recovery to latest from full + ptrack backups with loads when ptrack backup do""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, set_replication=True, @@ -456,7 +447,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_full_under_load_ptrack(self): """recovery to latest from full + page backups with loads when full backup do""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, set_replication=True, @@ -516,7 +506,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_to_xid_inclusive(self): """recovery with target inclusive false""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -575,7 +564,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_with_tablespace_mapping_1(self): """recovery using tablespace-mapping option""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -663,7 +651,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): def test_restore_with_tablespace_mapping_2(self): """recovery using tablespace-mapping option and page backup""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], diff --git a/tests/retention_test.py b/tests/retention_test.py index c30bdf93..265ed8da 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -19,7 +19,6 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -65,7 +64,6 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], diff --git a/tests/validate_test.py b/tests/validate_test.py index 80c793ab..f3fdf966 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -12,18 +12,14 @@ class ValidateTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(ValidateTest, self).__init__(*args, **kwargs) -# @classmethod -# def tearDownClass(cls): -# try: -# stop_all() -# except: -# pass + @classmethod + def tearDownClass(cls): + stop_all() # @unittest.skip("123") def test_validate_wal_1(self): """recovery to latest from full backup""" fname = self.id().split('.')[3] - print '\n {0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -62,19 +58,15 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.validate_pb(node, options=["--time='{:%Y-%m-%d %H:%M:%S}'".format( after_backup_time - timedelta(days=2))]) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal time") except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: Full backup satisfying target options is not found.\n' - ) + self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n') # Validate to unreal time #2 try: self.validate_pb(node, options=["--time='{:%Y-%m-%d %H:%M:%S}'".format( after_backup_time + timedelta(days=2))]) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal time") except ProbackupException, e: self.assertEqual( True, @@ -95,8 +87,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Validate to unreal xid try: self.validate_pb(node, options=["--xid=%d" % (int(target_xid) + 1000)]) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal xid") except ProbackupException, e: self.assertEqual( True, @@ -120,22 +111,16 @@ class ValidateTest(ProbackupTest, unittest.TestCase): try: self.validate_pb(node, id_backup, options=['--xid=%s' % target_xid]) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of wal segment corruption") except ProbackupException, e: - self.assertEqual( - True, - 'Possible WAL CORRUPTION' in e.message - ) + self.assertTrue(True, 'Possible WAL CORRUPTION' in e.message) try: self.validate_pb(node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of wal segment corruption") except ProbackupException, e: - self.assertEqual( - True, - 'Possible WAL CORRUPTION' in e.message - ) + self.assertTrue(True, 'Possible WAL CORRUPTION' in e.message) node.stop() @@ -143,7 +128,6 @@ class ValidateTest(ProbackupTest, unittest.TestCase): def test_validate_wal_lost_segment_1(self): """Loose segment which belong to some backup""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -167,19 +151,15 @@ class ValidateTest(ProbackupTest, unittest.TestCase): os.remove(os.path.join(self.backup_dir(node), "wal", wals[1])) try: self.validate_pb(node) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance") except ProbackupException, e: - self.assertEqual( - True, - 'is absent' in e.message - ) + self.assertTrue('is absent' in e.message) node.stop() + @unittest.expectedFailure def test_validate_wal_lost_segment_2(self): """Loose segment located between backups """ fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -209,7 +189,6 @@ class ValidateTest(ProbackupTest, unittest.TestCase): wals = map(int, wals) # delete last wal segment - print os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals))) os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))) # Need more accurate error message about loosing wal segment between backups From 7685097d2ed1d4c3db37ab55d7e26ae31d25936e Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 22 Jun 2017 15:10:52 +0300 Subject: [PATCH 03/37] Version 1.1.17 --- .gitignore | 17 +- COPYRIGHT | 3 +- Makefile | 99 ++-- configure.c | 126 ----- help.c | 238 ---------- pg_probackup.c | 285 ------------ src/archive.c | 106 +++++ backup.c => src/backup.c | 612 ++++++++++++++++++------- catalog.c => src/catalog.c | 59 ++- src/configure.c | 240 ++++++++++ data.c => src/data.c | 310 +++++++++---- delete.c => src/delete.c | 75 ++- dir.c => src/dir.c | 37 +- fetch.c => src/fetch.c | 0 src/help.c | 339 ++++++++++++++ init.c => src/init.c | 62 ++- parsexlog.c => src/parsexlog.c | 276 ++++++++--- src/pg_probackup.c | 475 +++++++++++++++++++ pg_probackup.h => src/pg_probackup.h | 137 ++++-- restore.c => src/restore.c | 106 +++-- show.c => src/show.c | 81 +++- status.c => src/status.c | 0 util.c => src/util.c | 10 +- src/utils/logger.c | 532 +++++++++++++++++++++ src/utils/logger.h | 44 ++ parray.c => src/utils/parray.c | 2 +- parray.h => src/utils/parray.h | 0 {pgut => src/utils}/pgut.c | 340 +++++++++----- {pgut => src/utils}/pgut.h | 46 +- validate.c => src/validate.c | 145 +++++- tests/__init__.py | 11 +- tests/backup_test.py | 212 +++++---- tests/class_check.py | 24 - tests/class_check1.py | 15 - tests/class_check2.py | 23 - tests/delete_test.py | 114 +++-- tests/expected/option_help.out | 66 ++- tests/expected/option_version.out | 2 +- tests/false_positive.py | 155 +++++++ tests/helpers/__init__.py | 2 + tests/{ => helpers}/ptrack_helpers.py | 163 ++++--- tests/init_test.py | 70 +-- tests/option_test.py | 196 ++++---- tests/pgpro560.py | 78 ++++ tests/pgpro589.py | 97 ++++ tests/pgpro688.py | 201 ++++++++ tests/ptrack_clean.py | 25 +- tests/ptrack_cluster.py | 76 +-- tests/ptrack_move_to_tablespace.py | 17 +- tests/ptrack_recovery.py | 17 +- tests/ptrack_vacuum.py | 17 +- tests/ptrack_vacuum_bits_frozen.py | 22 +- tests/ptrack_vacuum_bits_visibility.py | 22 +- tests/ptrack_vacuum_full.py | 32 +- tests/ptrack_vacuum_truncate.py | 22 +- tests/replica.py | 129 ++++++ tests/restore_test.py | 555 ++++++++++++---------- tests/retention_test.py | 62 +-- tests/show_test.py | 44 +- tests/validate_test.py | 284 ++++++++---- 60 files changed, 5377 insertions(+), 2178 deletions(-) delete mode 100644 configure.c delete mode 100644 help.c delete mode 100644 pg_probackup.c create mode 100644 src/archive.c rename backup.c => src/backup.c (76%) rename catalog.c => src/catalog.c (89%) create mode 100644 src/configure.c rename data.c => src/data.c (73%) rename delete.c => src/delete.c (84%) rename dir.c => src/dir.c (95%) rename fetch.c => src/fetch.c (100%) create mode 100644 src/help.c rename init.c => src/init.c (53%) rename parsexlog.c => src/parsexlog.c (69%) create mode 100644 src/pg_probackup.c rename pg_probackup.h => src/pg_probackup.h (83%) rename restore.c => src/restore.c (92%) rename show.c => src/show.c (71%) rename status.c => src/status.c (100%) rename util.c => src/util.c (95%) create mode 100644 src/utils/logger.c create mode 100644 src/utils/logger.h rename parray.c => src/utils/parray.c (99%) rename parray.h => src/utils/parray.h (100%) rename {pgut => src/utils}/pgut.c (81%) rename {pgut => src/utils}/pgut.h (82%) rename validate.c => src/validate.c (54%) delete mode 100644 tests/class_check.py delete mode 100644 tests/class_check1.py delete mode 100644 tests/class_check2.py create mode 100644 tests/false_positive.py create mode 100644 tests/helpers/__init__.py rename tests/{ => helpers}/ptrack_helpers.py (81%) create mode 100644 tests/pgpro560.py create mode 100644 tests/pgpro589.py create mode 100644 tests/pgpro688.py create mode 100644 tests/replica.py diff --git a/.gitignore b/.gitignore index 1c265564..417410c3 100644 --- a/.gitignore +++ b/.gitignore @@ -29,13 +29,14 @@ /tests/__pycache__/ /tests/tmp_dirs/ /tests/*pyc +/tests/helpers/*pyc # Extra files -/datapagemap.c -/datapagemap.h -/logging.h -/receivelog.c -/receivelog.h -/streamutil.c -/streamutil.h -/xlogreader.c +/src/datapagemap.c +/src/datapagemap.h +/src/logging.h +/src/receivelog.c +/src/receivelog.h +/src/streamutil.c +/src/streamutil.h +/src/xlogreader.c diff --git a/COPYRIGHT b/COPYRIGHT index dc7e2935..49d70472 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -1,4 +1,5 @@ -Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +Copyright (c) 2015-2017, Postgres Professional +Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California diff --git a/Makefile b/Makefile index 8cf6c669..e76a3ea9 100644 --- a/Makefile +++ b/Makefile @@ -1,54 +1,67 @@ PROGRAM = pg_probackup -OBJS = backup.o \ - catalog.o \ - configure.o \ - data.o \ - delete.o \ - dir.o \ - fetch.o \ - help.o \ - init.o \ - parray.o \ - pg_probackup.o \ - restore.o \ - show.o \ - status.o \ - util.o \ - validate.o \ - datapagemap.o \ - parsexlog.o \ - xlogreader.o \ - streamutil.o \ - receivelog.o \ - pgut/pgut.o +OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ + src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \ + src/pg_probackup.o src/restore.o src/show.o src/status.o \ + src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ + src/xlogreader.o src/streamutil.o src/receivelog.o \ + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o -EXTRA_CLEAN = datapagemap.c datapagemap.h xlogreader.c receivelog.c receivelog.h streamutil.c streamutil.h logging.h +EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h -all: checksrcdir datapagemap.h logging.h receivelog.h streamutil.h pg_probackup +all: checksrcdir src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h pg_probackup -MAKE_GLOBAL="../../src/Makefile.global" -TEST_GLOBAL:=$(shell test -e ../../src/Makefile.global) -ifeq ($(.SHELLSTATUS),1) +ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) - -.PHONY: checksrcdir -checksrcdir: ifndef top_srcdir @echo "You must have PostgreSQL source tree available to compile." @echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir" @echo "variable: \"make top_srcdir=\"" @exit 1 endif +# Those files are symlinked from the PostgreSQL sources. +src/xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/xlogreader.c + rm -f $@ && $(LN_S) $< ./src/xlogreader.c +src/datapagemap.c: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.c + rm -f $@ && $(LN_S) $< ./src/datapagemap.c +src/datapagemap.h: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.h + rm -f $@ && $(LN_S) $< src/datapagemap.h +src/logging.h: % : $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) $< ./src +src/receivelog.c: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.c + rm -f $@ && $(LN_S) $< ./src +src/receivelog.h: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.h + rm -f $@ && $(LN_S) $< ./src +src/streamutil.c: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.c + rm -f $@ && $(LN_S) $< ./src +src/streamutil.h: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.h + rm -f $@ && $(LN_S) $< ./src else -#TODO: fix me -REGRESS = subdir=contrib/pg_probackup top_builddir=../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk +# Those files are symlinked from the PostgreSQL sources. +src/xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/xlogreader.c + rm -f $@ && $(LN_S) ../$< ./src/xlogreader.c +src/datapagemap.c: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.c + rm -f $@ && $(LN_S) ../$< ./src/datapagemap.c +src/datapagemap.h: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.h + rm -f $@ && $(LN_S) ../$< src/datapagemap.h +src/logging.h: % : $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) ../$< ./src +src/receivelog.c: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.c + rm -f $@ && $(LN_S) ../$< ./src +src/receivelog.h: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.h + rm -f $@ && $(LN_S) ../$< ./src +src/streamutil.c: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.c + rm -f $@ && $(LN_S) ../$< ./src +src/streamutil.h: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.h + rm -f $@ && $(LN_S) ../$< ./src endif + PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} @@ -58,7 +71,7 @@ ifeq ($(PORTNAME), aix) endif envtest: - : top_srcdir=$(top_srcdir) + : top_srcdir=$( ) : libpq_srcdir = $(libpq_srcdir) # This rule's only purpose is to give the user instructions on how to pass @@ -71,23 +84,3 @@ ifndef top_srcdir @echo "variable: \"make top_srcdir=\"" @exit 1 endif - -# Those files are symlinked from the PostgreSQL sources. -xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% - rm -f $@ && $(LN_S) $< . -datapagemap.c: % : $(top_srcdir)/src/bin/pg_rewind/% - rm -f $@ && $(LN_S) $< . -datapagemap.h: % : $(top_srcdir)/src/bin/pg_rewind/% - rm -f && $(LN_S) $< . -#logging.c: % : $(top_srcdir)/src/bin/pg_rewind/% -# rm -f && $(LN_S) $< . -logging.h: % : $(top_srcdir)/src/bin/pg_rewind/% - rm -f && $(LN_S) $< . -receivelog.c: % : $(top_srcdir)/src/bin/pg_basebackup/% - rm -f && $(LN_S) $< . -receivelog.h: % : $(top_srcdir)/src/bin/pg_basebackup/% - rm -f && $(LN_S) $< . -streamutil.c: % : $(top_srcdir)/src/bin/pg_basebackup/% - rm -f && $(LN_S) $< . -streamutil.h: % : $(top_srcdir)/src/bin/pg_basebackup/% - rm -f && $(LN_S) $< . diff --git a/configure.c b/configure.c deleted file mode 100644 index fbd1b6b0..00000000 --- a/configure.c +++ /dev/null @@ -1,126 +0,0 @@ -/*------------------------------------------------------------------------- - * - * configure.c: - manage backup catalog. - * - * Portions Copyright (c) 2017-2017, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -/* Set configure options */ -int -do_configure(bool show_only) -{ - pgBackupConfig *config = readBackupCatalogConfigFile(); - if (pgdata) - config->pgdata = pgdata; - if (pgut_dbname) - config->pgdatabase = pgut_dbname; - if (host) - config->pghost = host; - if (port) - config->pgport = port; - if (username) - config->pguser = username; - - if (retention_redundancy) - config->retention_redundancy = retention_redundancy; - if (retention_window) - config->retention_window = retention_window; - - if (show_only) - writeBackupCatalogConfig(stderr, config); - else - writeBackupCatalogConfigFile(config); - - return 0; -} - -void -pgBackupConfigInit(pgBackupConfig *config) -{ - config->system_identifier = 0; - config->pgdata = NULL; - config->pgdatabase = NULL; - config->pghost = NULL; - config->pgport = NULL; - config->pguser = NULL; - - config->retention_redundancy = 0; - config->retention_window = 0; -} - -void -writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) -{ - fprintf(out, "#Backup instance info\n"); - fprintf(out, "PGDATA = %s\n", config->pgdata); - fprintf(out, "system-identifier = %li\n", config->system_identifier); - - fprintf(out, "#Connection parameters:\n"); - if (config->pgdatabase) - fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); - if (config->pghost) - fprintf(out, "PGHOST = %s\n", config->pghost); - if (config->pgport) - fprintf(out, "PGPORT = %s\n", config->pgport); - if (config->pguser) - fprintf(out, "PGUSER = %s\n", config->pguser); - - fprintf(out, "#Retention parameters:\n"); - if (config->retention_redundancy) - fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); - if (config->retention_window) - fprintf(out, "retention-window = %u\n", config->retention_window); - -} - -void -writeBackupCatalogConfigFile(pgBackupConfig *config) -{ - char path[MAXPGPATH]; - FILE *fp; - - join_path_components(path, backup_path, BACKUPS_DIR); - join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE); - fp = fopen(path, "wt"); - if (fp == NULL) - elog(ERROR, "cannot create %s: %s", - BACKUP_CATALOG_CONF_FILE, strerror(errno)); - - writeBackupCatalogConfig(fp, config); - fclose(fp); -} - - -pgBackupConfig* -readBackupCatalogConfigFile(void) -{ - pgBackupConfig *config = pgut_new(pgBackupConfig); - char path[MAXPGPATH]; - - pgut_option options[] = - { - /* configure options */ - { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, - { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, - { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, - { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, - { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, - { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, - { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, - { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, - {0} - }; - - join_path_components(path, backup_path, BACKUPS_DIR); - join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE); - - pgBackupConfigInit(config); - pgut_readopt(path, options, ERROR); - - return config; - -} diff --git a/help.c b/help.c deleted file mode 100644 index 9936ab6e..00000000 --- a/help.c +++ /dev/null @@ -1,238 +0,0 @@ -/*------------------------------------------------------------------------- - * - * help.c - * - * Portions Copyright (c) 2017-2017, Postgres Professional - * - *------------------------------------------------------------------------- - */ -#include "pg_probackup.h" - -static void help_init(void); -static void help_backup(void); -static void help_restore(void); -static void help_validate(void); -static void help_show(void); -static void help_delete(void); -static void help_set_config(void); -static void help_show_config(void); - -void -help_command(char *command) -{ - if (strcmp(command, "init") == 0) - help_init(); - else if (strcmp(command, "backup") == 0) - help_backup(); - else if (strcmp(command, "restore") == 0) - help_restore(); - else if (strcmp(command, "validate") == 0) - help_validate(); - else if (strcmp(command, "show") == 0) - help_show(); - else if (strcmp(command, "delete") == 0) - help_delete(); - else if (strcmp(command, "set-config") == 0) - help_set_config(); - else if (strcmp(command, "show-config") == 0) - help_show_config(); - else if (strcmp(command, "--help") == 0 - || strcmp(command, "help") == 0 - || strcmp(command, "-?") == 0 - || strcmp(command, "--version") == 0 - || strcmp(command, "version") == 0 - || strcmp(command, "-V") == 0) - printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); - else - printf(_("Unknown command. Try pg_probackup help\n")); - exit(0); -} - -void -help_pg_probackup(void) -{ - printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); - - printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); - - printf(_("\n %s version\n"), PROGRAM_NAME); - - printf(_("\n %s init -B backup-path -D pgdata-dir\n"), PROGRAM_NAME); - - printf(_("\n %s set-config -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]]\n")); - printf(_(" [--retention-window=retention-window]\n")); - - printf(_("\n %s show-config -B backup-dir\n"), PROGRAM_NAME); - - printf(_("\n %s backup -B backup-path -b backup-mode\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); - printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); - printf(_(" [--progress] [-q] [-v] [--delete-expired]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - - printf(_("\n %s restore -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); - - printf(_("\n %s validate -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline]\n")); - - printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-i backup-id]\n")); - - printf(_("\n %s delete -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [--wal] [-i backup-id | --expired]\n")); - - if ((PROGRAM_URL || PROGRAM_EMAIL)) - { - printf("\n"); - if (PROGRAM_URL) - printf("Read the website for details. <%s>\n", PROGRAM_URL); - if (PROGRAM_EMAIL) - printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); - } - exit(0); -} - -static void -help_init(void) -{ - printf(_("%s init -B backup-path -D pgdata-dir\n\n"), PROGRAM_NAME); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); -} - -static void -help_backup(void) -{ - printf(_("%s backup -B backup-path -b backup-mode\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); - printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); - printf(_(" [--progress] [-q] [-v] [--delete-expired]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|PTRACK\n")); - printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); - printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); - printf(_(" --stream stream the transaction log and include it in the backup\n")); - printf(_(" --archive-timeout wait timeout for WAL segment archiving\n")); - printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); - printf(_(" --backup-pg-log backup of pg_log directory\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --progress show progress\n")); - printf(_(" -q, --quiet don't write any messages\n")); - printf(_(" -v, --verbose verbose mode\n")); - printf(_(" --delete-expired delete backups expired according to current\n")); - printf(_(" retention policy after successful backup completion\n")); - - printf(_("\n Connection options:\n")); - printf(_(" -d, --dbname=DBNAME database to connect\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); - printf(_(" -p, --port=PORT database server port\n")); - printf(_(" -U, --username=USERNAME user name to connect as\n")); -} - -static void -help_restore(void) -{ - printf(_("%s restore -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); - printf(_(" -i, --backup-id=backup-id backup to restore\n")); - - printf(_(" --progress show progress\n")); - printf(_(" -q, --quiet don't write any messages\n")); - printf(_(" -v, --verbose verbose mode\n")); - printf(_(" --time=time time stamp up to which recovery will proceed\n")); - printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); - printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); - printf(_(" --timeline=timeline recovering into a particular timeline\n")); - printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); -} - -static void -help_validate(void) -{ - printf(_("%s validate -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); - printf(_(" -i, --backup-id=backup-id backup to validate\n")); - - printf(_(" --progress show progress\n")); - printf(_(" -q, --quiet don't write any messages\n")); - printf(_(" -v, --verbose verbose mode\n")); - printf(_(" --time=time time stamp up to which recovery will proceed\n")); - printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); - printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); - printf(_(" --timeline=timeline recovering into a particular timeline\n")); -} - -static void -help_show(void) -{ - printf(_("%s show -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-i backup-id]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); -} - -static void -help_delete(void) -{ - printf(_("%s delete -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [--wal] [-i backup-id | --expired]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --wal remove unnecessary wal files\n")); - printf(_(" -i, --backup-id=backup-id backup to delete\n")); - printf(_(" --expired delete backups expired according to current\n")); - printf(_(" retention policy\n")); -} - -static void -help_set_config(void) -{ - printf(_("%s set-config -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]]\n")); - printf(_(" [--retention-window=retention-window]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - - printf(_("\n Connection options:\n")); - printf(_(" -d, --dbname=DBNAME database to connect\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); - printf(_(" -p, --port=PORT database server port\n")); - printf(_(" -U, --username=USERNAME user name to connect as\n")); - - printf(_("\n Retention options:\n")); - printf(_(" --retention-redundancy=retention-redundancy\n")); - printf(_(" number of full backups to keep\n")); - printf(_(" --retention-window=retention-window\n")); - printf(_(" number of days of recoverability\n")); - -} - -static void -help_show_config(void) -{ - printf(_("%s show-config -B backup-dir\n\n"), PROGRAM_NAME); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); -} diff --git a/pg_probackup.c b/pg_probackup.c deleted file mode 100644 index deb2934b..00000000 --- a/pg_probackup.c +++ /dev/null @@ -1,285 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_probackup.c: Backup/Recovery manager for PostgreSQL. - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" -#include "streamutil.h" - -#include -#include -#include -#include - -const char *PROGRAM_VERSION = "1.1.11"; -const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; -const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; - -/* path configuration */ -char *backup_path; -char *pgdata; -char arclog_path[MAXPGPATH]; - -/* directory configuration */ -pgBackup current; -ProbackupSubcmd backup_subcmd; - -bool help = false; - -char *backup_id_string_param = NULL; -bool backup_logs = false; - -bool smooth_checkpoint; -int num_threads = 1; -bool stream_wal = false; -bool from_replica = false; -bool progress = false; -bool delete_wal = false; -bool delete_expired = false; -bool apply_to_all = false; -bool force_delete = false; -uint32 archive_timeout = 300; /* Wait timeout for WAL segment archiving */ - -uint64 system_identifier = 0; - -uint32 retention_redundancy = 0; -uint32 retention_window = 0; - -/* restore configuration */ -static char *target_time; -static char *target_xid; -static char *target_inclusive; -static TimeLineID target_tli; - -static void opt_backup_mode(pgut_option *opt, const char *arg); - -static pgut_option options[] = -{ - /* directory options */ - { 'b', 1, "help", &help, SOURCE_CMDLINE }, - { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, - { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, - /* common options */ - { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, - { 'b', 8, "stream", &stream_wal, SOURCE_CMDLINE }, - { 'b', 11, "progress", &progress, SOURCE_CMDLINE }, - { 's', 'i', "backup-id", &backup_id_string_param, SOURCE_CMDLINE }, - /* backup options */ - { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, - { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, - { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, - { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, - { 'u', 2, "archive-timeout", &archive_timeout, SOURCE_CMDLINE }, - { 'b', 19, "delete-expired", &delete_expired, SOURCE_CMDLINE }, - /* restore options */ - { 's', 3, "time", &target_time, SOURCE_CMDLINE }, - { 's', 4, "xid", &target_xid, SOURCE_CMDLINE }, - { 's', 5, "inclusive", &target_inclusive, SOURCE_CMDLINE }, - { 'u', 6, "timeline", &target_tli, SOURCE_CMDLINE }, - { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, - /* delete options */ - { 'b', 12, "wal", &delete_wal, SOURCE_CMDLINE }, - { 'b', 16, "expired", &delete_expired, SOURCE_CMDLINE }, - { 'b', 17, "all", &apply_to_all, SOURCE_CMDLINE }, - /* TODO not implemented yet */ - { 'b', 18, "force", &force_delete, SOURCE_CMDLINE }, - /* configure options */ - { 'u', 13, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, - { 'u', 14, "retention-window", &retention_window, SOURCE_CMDLINE }, - /* other */ - { 'U', 15, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, - - { 's', 'd', "pgdatabase" , &pgut_dbname, SOURCE_CMDLINE }, - { 's', 'h', "pghost" , &host, SOURCE_CMDLINE }, - { 's', 'p', "pgport" , &port, SOURCE_CMDLINE }, - { 's', 'U', "pguser" , &username, SOURCE_CMDLINE }, - { 'b', 'q', "quiet" , &quiet, SOURCE_CMDLINE }, - { 'b', 'v', "verbose" , &verbose, SOURCE_CMDLINE }, - { 'B', 'w', "no-password" , &prompt_password, SOURCE_CMDLINE }, - { 0 } -}; - -/* - * Entry point of pg_probackup command. - */ -int -main(int argc, char *argv[]) -{ - char path[MAXPGPATH]; - /* Check if backup_path is directory. */ - struct stat stat_buf; - int rc; - - /* initialize configuration */ - pgBackup_init(¤t); - - PROGRAM_NAME = get_progname(argv[0]); - set_pglocale_pgservice(argv[0], "pgscripts"); - - /* Parse subcommands and non-subcommand options */ - if (argc > 1) - { - if (strcmp(argv[1], "init") == 0) - backup_subcmd = INIT; - else if (strcmp(argv[1], "backup") == 0) - backup_subcmd = BACKUP; - else if (strcmp(argv[1], "restore") == 0) - backup_subcmd = RESTORE; - else if (strcmp(argv[1], "validate") == 0) - backup_subcmd = VALIDATE; - else if (strcmp(argv[1], "show") == 0) - backup_subcmd = SHOW; - else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE; - else if (strcmp(argv[1], "set-config") == 0) - backup_subcmd = SET_CONFIG; - else if (strcmp(argv[1], "show-config") == 0) - backup_subcmd = SHOW_CONFIG; - else if (strcmp(argv[1], "--help") == 0 - || strcmp(argv[1], "help") == 0 - || strcmp(argv[1], "-?") == 0) - { - if (argc > 2) - help_command(argv[2]); - else - help_pg_probackup(); - } - else if (strcmp(argv[1], "--version") == 0 - || strcmp(argv[1], "version") == 0 - || strcmp(argv[1], "-V") == 0) - { - if (argc == 2) - { - fprintf(stderr, "%s %s\n", PROGRAM_NAME, PROGRAM_VERSION); - exit(0); - } - else if (strcmp(argv[2], "--help") == 0) - help_command(argv[1]); - else - elog(ERROR, "Invalid arguments for \"%s\" subcommand", argv[1]); - } - else - elog(ERROR, "Unknown subcommand"); - } - - /* Parse command line arguments */ - pgut_getopt(argc, argv, options); - - if (help) - help_command(argv[2]); - - if (backup_path == NULL) - { - /* Try to read BACKUP_PATH from environment variable */ - backup_path = getenv("BACKUP_PATH"); - if (backup_path == NULL) - elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); - } - - rc = stat(backup_path, &stat_buf); - /* If rc == -1, there is no file or directory. So it's OK. */ - if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); - - /* Do not read options from file or env if we're going to set them */ - if (backup_subcmd != SET_CONFIG) - { - /* Read options from configuration file */ - join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE); - pgut_readopt(path, options, ERROR); - - /* Read environment variables */ - pgut_getopt_env(options); - } - - if (backup_id_string_param != NULL) - { - current.backup_id = base36dec(backup_id_string_param); - if (current.backup_id == 0) - elog(ERROR, "Invalid backup-id"); - } - - /* setup stream options */ - if (pgut_dbname != NULL) - dbname = pstrdup(pgut_dbname); - if (host != NULL) - dbhost = pstrdup(host); - if (port != NULL) - dbport = pstrdup(port); - if (username != NULL) - dbuser = pstrdup(username); - - /* path must be absolute */ - if (!is_absolute_path(backup_path)) - elog(ERROR, "-B, --backup-path must be an absolute path"); - if (pgdata != NULL && !is_absolute_path(pgdata)) - elog(ERROR, "-D, --pgdata must be an absolute path"); - - join_path_components(arclog_path, backup_path, "wal"); - - /* setup exclusion list for file search */ - if (!backup_logs) - { - int i; - - for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ - - /* Set 'pg_log' in first empty slot */ - pgdata_exclude_dir[i] = "pg_log"; - } - - if (target_time != NULL && target_xid != NULL) - elog(ERROR, "You can't specify recovery-target-time and recovery-target-xid at the same time"); - - if (num_threads < 1) - num_threads = 1; - - /* do actual operation */ - switch (backup_subcmd) - { - case INIT: - return do_init(); - case BACKUP: - return do_backup(); - case RESTORE: - return do_restore_or_validate(current.backup_id, - target_time, target_xid, - target_inclusive, target_tli, - true); - case VALIDATE: - return do_restore_or_validate(current.backup_id, - target_time, target_xid, - target_inclusive, target_tli, - false); - case SHOW: - return do_show(current.backup_id); - case DELETE: - if (delete_expired && backup_id_string_param) - elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); - if (delete_expired) - return do_retention_purge(); - else - return do_delete(current.backup_id); - case SHOW_CONFIG: - if (argc > 4) - elog(ERROR, "show-config command doesn't accept any options"); - return do_configure(true); - case SET_CONFIG: - if (argc == 4) - elog(ERROR, "set-config command requires at least one option"); - return do_configure(false); - } - - return 0; -} - -static void -opt_backup_mode(pgut_option *opt, const char *arg) -{ - current.backup_mode = parse_backup_mode(arg); -} diff --git a/src/archive.c b/src/archive.c new file mode 100644 index 00000000..1bb17643 --- /dev/null +++ b/src/archive.c @@ -0,0 +1,106 @@ +/*------------------------------------------------------------------------- + * + * archive.c: - pg_probackup specific archive commands for archive backups. + * + * + * Portions Copyright (c) 2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +#include +#include + +/* + * pg_probackup specific archive command for archive backups + * set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup + * --wal-file-path %p --wal-file-name %f', to move backups into arclog_path. + * Where archlog_path is $BACKUP_PATH/wal/system_id. + * Currently it just copies wal files to the new location. + * TODO: Planned options: compress, list the arclog content, + * compute and validate checksums. + */ +int +do_archive_push(char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + int64 system_id; + pgBackupConfig *config; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal_file_name %%f --wal_file_path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal_file_name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal_file_path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + /* verify that archive-push --instance parameter is valid */ + config = readBackupCatalogConfigFile(); + system_id = get_system_identifier(current_dir); + + if (config->pgdata == NULL) + elog(ERROR, "cannot read pg_probackup.conf for this instance"); + + if(system_id != config->system_identifier) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have SYSTEM_ID = %ld instead of %ld", + wal_file_name, instance_name, config->system_identifier, system_id); + + if (strcmp(current_dir, config->pgdata) != 0) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have PGDATA = %s instead of %s", + wal_file_name, instance_name, config->pgdata, current_dir); + + /* Create 'archlog_path' directory. Do nothing if it already exists. */ + dir_create_dir(arclog_path, DIR_PERMISSION); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + copy_wal_file(absolute_wal_file_path, backup_wal_file_path); + elog(INFO, "pg_probackup archive-push completed successfully"); + + return 0; +} + +/* + * pg_probackup specific restore command. + * Move files from arclog_path to pgdata/wal_file_path. + */ +int +do_archive_get(char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal_file_name %%f --wal_file_path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal_file_name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal_file_path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-get from %s to %s", backup_wal_file_path, absolute_wal_file_path); + copy_wal_file(backup_wal_file_path, absolute_wal_file_path); + elog(INFO, "pg_probackup archive-get completed successfully"); + + return 0; +} \ No newline at end of file diff --git a/backup.c b/src/backup.c similarity index 76% rename from backup.c rename to src/backup.c index f502bfd6..e8cb56e1 100644 --- a/backup.c +++ b/src/backup.c @@ -28,17 +28,38 @@ static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; + +/* + * How long we should wait for streaming end in seconds. + * Retreived as checkpoint_timeout + checkpoint_timeout * 0.1 + */ +static uint32 stream_stop_timeout = 0; +/* Time in which we started to wait for streaming end */ +static time_t stream_stop_begin = 0; + const char *progname = "pg_probackup"; /* list of files contained in backup */ static parray *backup_files_list = NULL; -static pthread_mutex_t check_stream_mut = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t start_stream_mut = PTHREAD_MUTEX_INITIALIZER; +/* + * We need to wait end of WAL streaming before execute pg_stop_backup(). + */ +static pthread_t stream_thread; static int is_ptrack_enable = false; -/* Backup connection */ +/* Backup connections */ static PGconn *backup_conn = NULL; +static PGconn *master_conn = NULL; + +/* PostgreSQL server version from "backup_conn" */ +static int server_version = 0; + +static bool exclusive_backup = false; +/* Is pg_start_backup() was executed */ +static bool backup_in_progress = false; typedef struct { @@ -61,10 +82,11 @@ static void do_backup_database(parray *backup_list); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); static void pg_switch_wal(void); static void pg_stop_backup(pgBackup *backup); +static int checkpoint_timeout(void); static void add_pgdata_files(parray *files, const char *root); static void write_backup_file_list(parray *files, const char *root); -static void wait_archive_lsn(XLogRecPtr lsn, bool prev_segno); +static void wait_wal_lsn(XLogRecPtr lsn); static void make_pagemap_from_ptrack(parray *files); static void StreamLog(void *arg); @@ -73,6 +95,7 @@ static void pg_ptrack_clear(void); static bool pg_ptrack_support(void); static bool pg_ptrack_enable(void); static bool pg_is_in_recovery(void); +static bool pg_archive_enabled(void); static char *pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_oid, @@ -104,7 +127,6 @@ do_backup_database(parray *backup_list) XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; pthread_t backup_threads[num_threads]; - pthread_t stream_thread; backup_files_args *backup_threads_args[num_threads]; pgBackup *prev_backup = NULL; @@ -141,29 +163,11 @@ do_backup_database(parray *backup_list) strncat(label, " with pg_probackup", lengthof(label)); pg_start_backup(label, smooth_checkpoint, ¤t); - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - - /* start stream replication */ - if (stream_wal) - { - join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); - dir_create_dir(dst_backup_path, DIR_PERMISSION); - - pthread_mutex_lock(&check_stream_mut); - pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, dst_backup_path); - pthread_mutex_lock(&check_stream_mut); - if (conn == NULL) - elog(ERROR, "Cannot continue backup because stream connect has failed."); - - pthread_mutex_unlock(&check_stream_mut); - } - /* * If backup_label does not exist in $PGDATA, stop taking backup. * NOTE. We can check it only on master, though. */ - if(!from_replica) + if (exclusive_backup) { char label_path[MAXPGPATH]; join_path_components(label_path, pgdata, PG_BACKUP_LABEL_FILE); @@ -177,6 +181,24 @@ do_backup_database(parray *backup_list) } } + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + /* start stream replication */ + if (stream_wal) + { + join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); + dir_create_dir(dst_backup_path, DIR_PERMISSION); + + pthread_mutex_lock(&start_stream_mut); + pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, dst_backup_path); + pthread_mutex_lock(&start_stream_mut); + if (conn == NULL) + elog(ERROR, "Cannot continue backup because stream connect has failed."); + + pthread_mutex_unlock(&start_stream_mut); + } + /* * To take incremental backup get the filelist of the last completed database */ @@ -215,16 +237,6 @@ do_backup_database(parray *backup_list) */ if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) { - /* - * Switch to a new WAL segment. It is necessary to get archived WAL - * segment, which includes start LSN of current backup. - * - * Do not switch for standby node and if backup is stream. - */ - if (!from_replica && !stream_wal) - pg_switch_wal(); - if (!stream_wal) - wait_archive_lsn(current.start_lsn, false); /* * Build the page map. Obtain information about changed pages * reading WAL segments present in archives up to the point @@ -276,8 +288,7 @@ do_backup_database(parray *backup_list) char dirpath[MAXPGPATH]; char *dir_name = GetRelativePath(file->path, pgdata); - if (verbose) - elog(LOG, "Create directory \"%s\"", dir_name); + elog(LOG, "Create directory \"%s\"", dir_name); join_path_components(dirpath, database_path, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); @@ -305,12 +316,11 @@ do_backup_database(parray *backup_list) /* Run threads */ for (i = 0; i < num_threads; i++) { - if (verbose) - elog(WARNING, "Start thread num:%li", parray_num(backup_threads_args[i]->backup_files_list)); + elog(LOG, "Start thread num:%li", parray_num(backup_threads_args[i]->backup_files_list)); pthread_create(&backup_threads[i], NULL, (void *(*)(void *)) backup_files, backup_threads_args[i]); } - /* Wait theads */ + /* Wait threads */ for (i = 0; i < num_threads; i++) { pthread_join(backup_threads[i], NULL); @@ -333,9 +343,6 @@ do_backup_database(parray *backup_list) parray *xlog_files_list; char pg_xlog_path[MAXPGPATH]; - /* Wait for the completion of stream */ - pthread_join(stream_thread, NULL); - /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); @@ -367,11 +374,12 @@ do_backup_database(parray *backup_list) { pgFile *file = (pgFile *) parray_get(backup_files_list, i); - if (!S_ISREG(file->mode)) - continue; + if (S_ISDIR(file->mode)) + current.data_bytes += 4096; /* Count the amount of the data actually copied */ - current.data_bytes += file->write_size; + if (S_ISREG(file->mode)) + current.data_bytes += file->write_size; } if (backup_files_list) @@ -400,13 +408,15 @@ do_backup(void) backup_conn = pgut_connect(pgut_dbname); pgut_atexit_push(backup_disconnect, NULL); - /* Confirm that this server version is supported */ - check_server_version(); /* Confirm data block size and xlog block size are compatible */ confirm_block_size("block_size", BLCKSZ); confirm_block_size("wal_block_size", XLOG_BLCKSZ); from_replica = pg_is_in_recovery(); + + /* Confirm that this server version is supported */ + check_server_version(); + current.checksum_version = get_data_checksum_version(true); current.stream = stream_wal; @@ -422,6 +432,21 @@ do_backup(void) elog(ERROR, "Ptrack is disabled"); } + /* archiving check */ + if (!current.stream && !pg_archive_enabled()) + elog(ERROR, "Archiving must be enabled for archive backup"); + + if (from_replica) + { + /* Check master connection options */ + if (master_host == NULL) + elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); + + /* Create connection to master server */ + master_conn = pgut_connect_extended(master_host, master_port, + master_db, master_user, password); + } + /* Get exclusive lock of backup catalog */ catalog_lock(); @@ -458,6 +483,13 @@ do_backup(void) do_backup_database(backup_list); pgut_atexit_pop(backup_cleanup, NULL); + /* compute size of wal files of this backup stored in the archive */ + if (!current.stream) + { + current.wal_bytes = XLOG_SEG_SIZE * + (current.stop_lsn/XLogSegSize - current.start_lsn/XLogSegSize + 1); + } + /* Backup is done. Update backup status */ current.end_time = time(NULL); current.status = BACKUP_STATUS_DONE; @@ -486,7 +518,6 @@ do_backup(void) static void check_server_version(void) { - static int server_version = 0; /* confirm server version */ server_version = PQserverVersion(backup_conn); @@ -503,6 +534,9 @@ check_server_version(void) server_version / 10000, (server_version / 100) % 100, server_version % 100, "9.6"); + + /* Do exclusive backup only for PostgreSQL 9.5 */ + exclusive_backup = server_version < 90600; } /* @@ -519,7 +553,7 @@ check_system_identifiers(void) uint64 system_id_pgdata; char *val; - system_id_pgdata = get_system_identifier(); + system_id_pgdata = get_system_identifier(pgdata); res = pgut_execute(backup_conn, "SELECT system_identifier FROM pg_control_system()", @@ -578,7 +612,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; - if (from_replica) + if (!exclusive_backup) res = pgut_execute(backup_conn, "SELECT pg_start_backup($1, $2, false)", 2, @@ -589,12 +623,30 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) 2, params); + backup_in_progress = true; + /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); /* Calculate LSN */ backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; PQclear(res); + + /* + * Switch to a new WAL segment. It is necessary to get archived WAL + * segment, which includes start LSN of current backup. + * + * Do not switch for standby node and if backup is stream. + */ + if (!from_replica && !stream_wal) + pg_switch_wal(); + if (!stream_wal) + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + wait_wal_lsn(backup->start_lsn); } /* @@ -610,19 +662,19 @@ pg_switch_wal(void) NULL); PQclear(res); -#if PG_VERSION_NUM >= 100000 - res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_wal()", 0, - NULL); -#else - res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_xlog()", 0, - NULL); -#endif + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_wal()", 0, + NULL); + else + res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_xlog()", 0, + NULL); + PQclear(res); } /* * Check if the instance supports ptrack - * TODO Implement check of ptrack_version() instead of existing one + * TODO Maybe we should rather check ptrack_version()? */ static bool pg_ptrack_support(void) @@ -677,6 +729,25 @@ pg_is_in_recovery(void) return false; } +/* + * Check if archiving is enabled + */ +static bool +pg_archive_enabled(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show archive_mode", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "off") == 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + /* Clear ptrack files in all databases of the instance we connected to */ static void pg_ptrack_clear(void) @@ -757,33 +828,61 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_oid, } /* - * Wait for target 'lsn' to be archived in archive 'wal' directory with - * WAL segment file. + * Wait for target 'lsn'. + * + * If current backup started in archive mode wait for 'lsn' to be archived in + * archive 'wal' directory with WAL segment file. + * If current backup started in stream mode wait for 'lsn' to be streamed in + * 'pg_xlog' directory. */ static void -wait_archive_lsn(XLogRecPtr lsn, bool prev_segno) +wait_wal_lsn(XLogRecPtr lsn) { TimeLineID tli; XLogSegNo targetSegNo; - char wal_path[MAXPGPATH]; - char wal_file[MAXFNAMELEN]; - uint32 try_count = 0; - - Assert(!stream_wal); + char wal_dir[MAXPGPATH], + wal_segment_full_path[MAXPGPATH]; + char wal_segment[MAXFNAMELEN]; + uint32 try_count = 0, + timeout; tli = get_current_timeline(false); /* Compute the name of the WAL file containig requested LSN */ XLByteToSeg(lsn, targetSegNo); - if (prev_segno) - targetSegNo--; - XLogFileName(wal_file, tli, targetSegNo); + XLogFileName(wal_segment, tli, targetSegNo); - join_path_components(wal_path, arclog_path, wal_file); - - /* Wait until switched WAL is archived */ - while (!fileExists(wal_path)) + if (stream_wal) { + pgBackupGetPath2(¤t, wal_dir, lengthof(wal_dir), + DATABASE_DIR, PG_XLOG_DIR); + join_path_components(wal_segment_full_path, wal_dir, wal_segment); + + timeout = (uint32) checkpoint_timeout(); + timeout = timeout + timeout * 0.1; + } + else + { + join_path_components(wal_segment_full_path, arclog_path, wal_segment); + timeout = archive_timeout; + } + + /* Wait until target LSN is archived or streamed */ + while (true) + { + bool file_exists = fileExists(wal_segment_full_path); + + if (file_exists) + { + /* + * A WAL segment found. Check LSN on it. + */ + if ((stream_wal && wal_contains_lsn(wal_dir, lsn, tli)) || + (!stream_wal && wal_contains_lsn(arclog_path, lsn, tli))) + /* Target LSN was found */ + return; + } + sleep(1); if (interrupted) elog(ERROR, "interrupted during waiting for WAL archiving"); @@ -792,21 +891,20 @@ wait_archive_lsn(XLogRecPtr lsn, bool prev_segno) /* Inform user if WAL segment is absent in first attempt */ if (try_count == 1) elog(INFO, "wait for LSN %X/%X in archived WAL segment %s", - (uint32) (lsn >> 32), (uint32) lsn, wal_path); + (uint32) (lsn >> 32), (uint32) lsn, wal_segment_full_path); - if (archive_timeout > 0 && try_count > archive_timeout) - elog(ERROR, - "switched WAL segment %s could not be archived in %d seconds", - wal_file, archive_timeout); + if (timeout > 0 && try_count > timeout) + { + if (file_exists) + elog(ERROR, "WAL segment %s was archived, " + "but target LSN %X/%X could not be archived in %d seconds", + wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); + else + elog(ERROR, + "switched WAL segment %s could not be archived in %d seconds", + wal_segment, timeout); + } } - - /* - * WAL segment was archived. Check LSN on it if we waited current WAL - * segment, not previous. - */ - if (!prev_segno && !wal_contains_lsn(arclog_path, lsn, tli)) - elog(ERROR, "WAL segment %s doesn't contain target LSN %X/%X", - wal_file, (uint32) (lsn >> 32), (uint32) lsn); } /* @@ -818,6 +916,9 @@ pg_stop_backup(pgBackup *backup) PGresult *res; uint32 xlogid; uint32 xrecoff; + XLogRecPtr restore_lsn = InvalidXLogRecPtr; + bool sent = false; + int pg_stop_backup_timeout = 0; /* * We will use this values if there are no transactions between start_lsn @@ -826,36 +927,173 @@ pg_stop_backup(pgBackup *backup) time_t recovery_time; TransactionId recovery_xid; + if (!backup_in_progress) + elog(FATAL, "backup is not in progress"); + /* Remove annoying NOTICE messages generated by backend */ res = pgut_execute(backup_conn, "SET client_min_messages = warning;", 0, NULL); PQclear(res); - if (from_replica) + /* Create restore point */ + if (backup != NULL) + { + const char *params[1]; + char name[1024]; + char *backup_id; + + backup_id = base36enc(backup->start_time); + + if (!from_replica) + { + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + backup_id); + params[0] = name; + + res = pgut_execute(backup_conn, "SELECT pg_create_restore_point($1)", + 1, params); + PQclear(res); + } + else + { + uint32 try_count = 0; + + snprintf(name, lengthof(name), "pg_probackup, backup_id %s. Replica Backup", + backup_id); + params[0] = name; + + res = pgut_execute(master_conn, "SELECT pg_create_restore_point($1)", + 1, params); + /* Extract timeline and LSN from result */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + restore_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + PQclear(res); + + /* Wait for restore_lsn from master */ + while (true) + { + XLogRecPtr min_recovery_lsn; + + res = pgut_execute(backup_conn, "SELECT min_recovery_end_location from pg_control_recovery()", + 0, NULL); + /* Extract timeline and LSN from result */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + min_recovery_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + PQclear(res); + + /* restore_lsn was streamed and applied to the replica */ + if (min_recovery_lsn >= restore_lsn) + break; + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for restore point LSN"); + try_count++; + + /* Inform user if restore_lsn is absent in first attempt */ + if (try_count == 1) + elog(INFO, "Wait for restore point LSN %X/%X to be streamed " + "to replica", + (uint32) (restore_lsn >> 32), (uint32) restore_lsn); + + if (replica_timeout > 0 && try_count > replica_timeout) + elog(ERROR, "Restore point LSN %X/%X could not be " + "streamed to replica in %d seconds", + (uint32) (restore_lsn >> 32), (uint32) restore_lsn, + replica_timeout); + } + } + + pfree(backup_id); + } + + /* + * send pg_stop_backup asynchronously because we could came + * here from backup_cleanup() after some error caused by + * postgres archive_command problem and in this case we will + * wait for pg_stop_backup() forever. + */ + if (!exclusive_backup) /* * Stop the non-exclusive backup. Besides stop_lsn it returns from * pg_stop_backup(false) copy of the backup label and tablespace map * so they can be written to disk by the caller. */ - res = pgut_execute(backup_conn, + sent = pgut_send(backup_conn, "SELECT *, txid_snapshot_xmax(txid_current_snapshot())," " current_timestamp(0)::timestamp" " FROM pg_stop_backup(false)", - 0, NULL); + 0, NULL, WARNING); else - res = pgut_execute(backup_conn, + sent = pgut_send(backup_conn, "SELECT *, txid_snapshot_xmax(txid_current_snapshot())," " current_timestamp(0)::timestamp" " FROM pg_stop_backup()", - 0, NULL); + 0, NULL, WARNING); + + if (!sent) + elog(WARNING, "Failed to send pg_stop_backup query"); + + + /* + * Wait for the result of pg_stop_backup(), + * but no longer than PG_STOP_BACKUP_TIMEOUT seconds + */ + elog(INFO, "wait for pg_stop_backup()"); + + while (1) + { + if (!PQconsumeInput(backup_conn) || PQisBusy(backup_conn)) + { + pg_stop_backup_timeout++; + sleep(1); + + if (interrupted) + { + pgut_cancel(backup_conn); + elog(ERROR, "interrupted during waiting for pg_stop_backup"); + } + /* + * If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds, + * send an interrupt. + */ + if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT) + { + pgut_cancel(backup_conn); + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", + PG_STOP_BACKUP_TIMEOUT); + } + } + else + { + res = PQgetResult(backup_conn); + break; + } + } + + if (!res) + elog(ERROR, "pg_stop backup() failed"); + + backup_in_progress = false; /* Extract timeline and LSN from results of pg_stop_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); /* Calculate LSN */ stop_backup_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + if (!XRecOffIsValid(stop_backup_lsn)) + { + stop_backup_lsn = restore_lsn; + } + + if (!XRecOffIsValid(stop_backup_lsn)) + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + /* Write backup_label and tablespace_map for backup from replica */ - if (from_replica) + if (!exclusive_backup) { char path[MAXPGPATH]; char backup_label[MAXPGPATH]; @@ -876,12 +1114,18 @@ pg_stop_backup(pgBackup *backup) fwrite(PQgetvalue(res, 0, 1), 1, strlen(PQgetvalue(res, 0, 1)), fp); fclose(fp); - /* TODO What for do we save the file into backup_list? */ - file = pgFileNew(backup_label, true); - calc_file_checksum(file); - free(file->path); - file->path = strdup(PG_BACKUP_LABEL_FILE); - parray_append(backup_files_list, file); + /* + * It's vital to check if backup_files_list is initialized, + * because we could get here because the backup was interrupted + */ + if (backup_files_list) + { + file = pgFileNew(backup_label, true); + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_BACKUP_LABEL_FILE); + parray_append(backup_files_list, file); + } /* Write tablespace_map */ if (strlen(PQgetvalue(res, 0, 2)) > 0) @@ -927,8 +1171,9 @@ pg_stop_backup(pgBackup *backup) PQclear(res); - if (!stream_wal) - wait_archive_lsn(stop_backup_lsn, false); + if (stream_wal) + /* Wait for the completion of stream */ + pthread_join(stream_thread, NULL); /* Fill in fields if that is the correct end of backup. */ if (backup != NULL) @@ -936,9 +1181,17 @@ pg_stop_backup(pgBackup *backup) char *xlog_path, stream_xlog_path[MAXPGPATH]; + /* + * Wait for stop_lsn to be archived or streamed. + * We wait for stop_lsn in stream mode just in case. + */ + wait_wal_lsn(stop_backup_lsn); + if (stream_wal) { - join_path_components(stream_xlog_path, pgdata, PG_XLOG_DIR); + pgBackupGetPath2(backup, stream_xlog_path, + lengthof(stream_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); xlog_path = stream_xlog_path; } else @@ -957,6 +1210,33 @@ pg_stop_backup(pgBackup *backup) } } +/* + * Retreive checkpoint_timeout GUC value in seconds. + */ +static int +checkpoint_timeout(void) +{ + PGresult *res; + const char *val; + const char *hintmsg; + int val_int; + + res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); + val = PQgetvalue(res, 0, 0); + PQclear(res); + + if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + { + if (hintmsg) + elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, + hintmsg); + else + elog(ERROR, "Invalid value of checkout_timeout %s", val); + } + + return val_int; +} + /* * Return true if the path is a existing regular file. */ @@ -981,16 +1261,6 @@ fileExists(const char *path) static void backup_cleanup(bool fatal, void *userdata) { - char path[MAXPGPATH]; - - /* If backup_label exists in $PGDATA, notify stop of backup to PostgreSQL */ - join_path_components(path, pgdata, PG_BACKUP_LABEL_FILE); - if (fileExists(path)) - { - elog(LOG, "%s exists, stop backup", PG_BACKUP_LABEL_FILE); - pg_stop_backup(NULL); /* don't care stop_lsn on error case */ - } - /* * Update status of backup in BACKUP_CONTROL_FILE to ERROR. * end_time != 0 means backup finished @@ -1002,6 +1272,15 @@ backup_cleanup(bool fatal, void *userdata) current.status = BACKUP_STATUS_ERROR; pgBackupWriteBackupControlFile(¤t); } + + /* + * If backup is in progress, notify stop of backup to PostgreSQL + */ + if (backup_in_progress) + { + elog(LOG, "backup in progress, stop backup"); + pg_stop_backup(NULL); /* don't care stop_lsn on error case */ + } } /* @@ -1011,6 +1290,8 @@ static void backup_disconnect(bool fatal, void *userdata) { pgut_disconnect(backup_conn); + if (master_conn) + pgut_disconnect(master_conn); } /* Count bytes in file */ @@ -1089,7 +1370,6 @@ backup_compressed_file_partially(pgFile *file, void *arg, size_t *skip_size) * verify checksum and copy. * In incremental backup mode, copy only files or datafiles' pages changed after * previous backup. - * TODO review */ static void backup_files(void *arg) @@ -1144,32 +1424,11 @@ backup_files(void *arg) if (S_ISREG(buf.st_mode)) { - /* skip files which have not been modified since last backup */ - /* TODO Implement: compare oldfile and newfile checksum. Now it's just a stub */ - if (arguments->prev_backup_filelist) - { - pgFile *prev_file = NULL; - pgFile **p = (pgFile **) parray_bsearch(arguments->prev_backup_filelist, - file, pgFileComparePath); - if (p) - prev_file = *p; - - if (prev_file && false) - { - file->write_size = BYTES_INVALID; - if (verbose) - elog(LOG, "File \"%s\" has not changed since previous backup", - file->path); - continue; - } - } - /* copy the file into backup */ if (file->is_datafile) { - if (is_compressed_data_file(file)) + if (file->is_cfs) { - /* TODO review */ size_t skip_size = 0; if (backup_compressed_file_partially(file, arguments, &skip_size)) { @@ -1178,9 +1437,7 @@ backup_files(void *arg) arguments->to_root, file, skip_size)) { - /* record as skipped file in file_xxx.txt */ file->write_size = BYTES_INVALID; - elog(LOG, "skip"); continue; } } @@ -1188,9 +1445,7 @@ backup_files(void *arg) arguments->to_root, file)) { - /* record as skipped file in file_xxx.txt */ file->write_size = BYTES_INVALID; - elog(LOG, "skip"); continue; } } @@ -1222,7 +1477,6 @@ backup_files(void *arg) /* * Append files to the backup list array. - * TODO review */ static void add_pgdata_files(parray *files, const char *root) @@ -1247,7 +1501,7 @@ add_pgdata_files(parray *files, const char *root) /* data files are under "base", "global", or "pg_tblspc" */ relative = GetRelativePath(file->path, root); if (!path_is_prefix_of_path("base", relative) && - /*!path_is_prefix_of_path("global", relative) &&*/ //TODO What's wrong with this line? + !path_is_prefix_of_path("global", relative) && !path_is_prefix_of_path(PG_TBLSPC_DIR, relative)) continue; @@ -1477,7 +1731,10 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) pg_free(rel_path); } -/* TODO review it */ +/* + * Given a list of files in the instance to backup, build a pagemap for each + * data file that has ptrack. Result is saved in the pagemap field of pgFile. + */ static void make_pagemap_from_ptrack(parray *files) { @@ -1499,6 +1756,8 @@ make_pagemap_from_ptrack(parray *files) size_t ptrack_nonparsed_size = 0; size_t start_addr; + /* Compute db_oid and rel_oid of the relation from the path */ + tablespace = strstr(p->ptrack_path, PG_TBLSPC_DIR); if (tablespace) @@ -1524,19 +1783,25 @@ make_pagemap_from_ptrack(parray *files) p->path); sscanf(p->path + sep_iter + 1, "%u/%u", &db_oid, &rel_oid); - + + /* get ptrack map for all segments of the relation in a raw format */ ptrack_nonparsed = pg_ptrack_get_and_clear(tablespace_oid, db_oid, rel_oid, &ptrack_nonparsed_size); - /* TODO What is 8? */ - start_addr = (RELSEG_SIZE/8)*p->segno; - if (start_addr + RELSEG_SIZE/8 > ptrack_nonparsed_size) + /* + * FIXME When do we cut VARHDR from ptrack_nonparsed? + * Compute the beginning of the ptrack map related to this segment + */ + start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*p->segno; + + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) p->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; else - p->pagemap.bitmapsize = RELSEG_SIZE/8; + p->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize); memcpy(p->pagemap.bitmap, ptrack_nonparsed+start_addr, p->pagemap.bitmapsize); + pg_free(ptrack_nonparsed); } } @@ -1554,10 +1819,9 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) static XLogRecPtr prevpos = InvalidXLogRecPtr; /* we assume that we get called once at the end of each segment */ - if (verbose && segment_finished) - fprintf(stderr, _("%s: finished segment at %X/%X (timeline %u)\n"), - PROGRAM_NAME, (uint32) (xlogpos >> 32), (uint32) xlogpos, - timeline); + if (segment_finished) + elog(LOG, _("finished segment at %X/%X (timeline %u)\n"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); /* * Note that we report the previous, not current, position here. After a @@ -1568,12 +1832,31 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) * timeline, but it's close enough for reporting purposes. */ if (prevtimeline != 0 && prevtimeline != timeline) - fprintf(stderr, _("%s: switched to timeline %u at %X/%X\n"), - PROGRAM_NAME, timeline, - (uint32) (prevpos >> 32), (uint32) prevpos); + elog(LOG, _("switched to timeline %u at %X/%X\n"), + timeline, (uint32) (prevpos >> 32), (uint32) prevpos); - if (stop_backup_lsn != InvalidXLogRecPtr && xlogpos > stop_backup_lsn) - return true; + if (!XLogRecPtrIsInvalid(stop_backup_lsn)) + { + if (xlogpos > stop_backup_lsn) + return true; + + /* pg_stop_backup() was executed, wait for the completion of stream */ + if (stream_stop_timeout == 0) + { + elog(INFO, "Wait for LSN %X/%X to be streamed", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); + + stream_stop_timeout = checkpoint_timeout(); + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + + stream_stop_begin = time(NULL); + } + + if (time(NULL) - stream_stop_begin > stream_stop_timeout) + elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, + stream_stop_timeout); + } prevtimeline = timeline; prevpos = xlogpos; @@ -1598,7 +1881,7 @@ StreamLog(void *arg) conn = GetConnection(); if (!conn) { - pthread_mutex_unlock(&check_stream_mut); + pthread_mutex_unlock(&start_stream_mut); /* Error message already written in GetConnection() */ return; } @@ -1622,7 +1905,7 @@ StreamLog(void *arg) disconnect_and_exit(1); /* Ok we have normal stream connect and main process can work again */ - pthread_mutex_unlock(&check_stream_mut); + pthread_mutex_unlock(&start_stream_mut); /* * We must use startpos as start_lsn from start_backup @@ -1634,14 +1917,15 @@ StreamLog(void *arg) */ startpos -= startpos % XLOG_SEG_SIZE; + /* Initialize timeout */ + stream_stop_timeout = 0; + stream_stop_begin = 0; + /* * Start the replication */ - if (verbose) - fprintf(stderr, - _("%s: starting log streaming at %X/%X (timeline %u)\n"), - PROGRAM_NAME, (uint32) (startpos >> 32), (uint32) startpos, - starttli); + elog(LOG, _("starting log streaming at %X/%X (timeline %u)\n"), + (uint32) (startpos >> 32), (uint32) startpos, starttli); #if PG_VERSION_NUM >= 90600 { @@ -1656,13 +1940,13 @@ StreamLog(void *arg) ctl.synchronous = false; ctl.mark_done = false; if(ReceiveXlogStream(conn, &ctl) == false) - elog(ERROR, "Problem in recivexlog"); + elog(ERROR, "Problem in receivexlog"); } #else if(ReceiveXlogStream(conn, startpos, starttli, NULL, basedir, stop_streaming, standby_message_timeout, NULL, false, false) == false) - elog(ERROR, "Problem in recivexlog"); + elog(ERROR, "Problem in receivexlog"); #endif PQfinish(conn); @@ -1673,7 +1957,7 @@ StreamLog(void *arg) * cfs_mmap() and cfs_munmap() function definitions mirror ones * from cfs.h, but doesn't use atomic variables, since they are * not allowed in frontend code. - * TODO Is it so? + * * Since we cannot take atomic lock on files compressed by CFS, * it should care about not changing files while backup is running. */ diff --git a/catalog.c b/src/catalog.c similarity index 89% rename from catalog.c rename to src/catalog.c index 5583dfdd..1b5cc383 100644 --- a/catalog.c +++ b/src/catalog.c @@ -50,7 +50,7 @@ catalog_lock(void) pid_t my_pid, my_p_pid; - join_path_components(lock_file, backup_path, BACKUP_CATALOG_PID); + join_path_components(lock_file, backup_instance_path, BACKUP_CATALOG_PID); /* * If the PID in the lockfile is our own PID or our parent's or @@ -84,7 +84,7 @@ catalog_lock(void) /* * We need a loop here because of race conditions. But don't loop forever - * (for example, a non-writable $backup_path directory might cause a failure + * (for example, a non-writable $backup_instance_path directory might cause a failure * that won't go away). 100 tries seems like plenty. */ for (ntries = 0;; ntries++) @@ -253,16 +253,14 @@ catalog_get_backup_list(time_t requested_backup_id) { DIR *date_dir = NULL; struct dirent *date_ent = NULL; - char backups_path[MAXPGPATH]; parray *backups = NULL; pgBackup *backup = NULL; - /* open backup root directory */ - join_path_components(backups_path, backup_path, BACKUPS_DIR); - date_dir = opendir(backups_path); + /* open backup instance backups directory */ + date_dir = opendir(backup_instance_path); if (date_dir == NULL) { - elog(WARNING, "cannot open directory \"%s\": %s", backups_path, + elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, strerror(errno)); goto err_proc; } @@ -275,12 +273,12 @@ catalog_get_backup_list(time_t requested_backup_id) char date_path[MAXPGPATH]; /* skip not-directory entries and hidden entries */ - if (!IsDir(backups_path, date_ent->d_name) + if (!IsDir(backup_instance_path, date_ent->d_name) || date_ent->d_name[0] == '.') continue; /* open subdirectory of specific backup */ - join_path_components(date_path, backups_path, date_ent->d_name); + join_path_components(date_path, backup_instance_path, date_ent->d_name); /* read backup information from BACKUP_CONTROL_FILE */ snprintf(backup_conf_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONTROL_FILE); @@ -309,7 +307,7 @@ catalog_get_backup_list(time_t requested_backup_id) if (errno) { elog(WARNING, "cannot read backup root directory \"%s\": %s", - backups_path, strerror(errno)); + backup_instance_path, strerror(errno)); goto err_proc; } @@ -388,6 +386,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "#Configuration\n"); fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); fprintf(out, "stream = %s\n", backup->stream?"true":"false"); + fprintf(out, "compress-alg = %s\n", deparse_compress_alg(compress_alg)); + fprintf(out, "compress-level = %d\n", compress_level); + fprintf(out, "from-replica = %s\n", from_replica?"true":"false"); fprintf(out, "\n#Compatibility\n"); fprintf(out, "block-size = %u\n", backup->block_size); @@ -397,11 +398,11 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "\n#Result backup info\n"); fprintf(out, "timelineid = %d\n", backup->tli); /* LSN returned by pg_start_backup */ - fprintf(out, "start-lsn = %x/%08x\n", + fprintf(out, "start-lsn = %X/%X\n", (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); /* LSN returned by pg_stop_backup */ - fprintf(out, "stop-lsn = %x/%08x\n", + fprintf(out, "stop-lsn = %X/%X\n", (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); @@ -426,6 +427,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->data_bytes != BYTES_INVALID) fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + if (backup->data_bytes != BYTES_INVALID) + fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + fprintf(out, "status = %s\n", status2str(backup->status)); /* 'parent_backup' is set if it is incremental backup */ @@ -470,6 +474,9 @@ readBackupControlFile(const char *path) char *stop_lsn = NULL; char *status = NULL; char *parent_backup = NULL; + char *compress_alg = NULL; + int *compress_level; + bool *from_replica; pgut_option options[] = { @@ -482,12 +489,16 @@ readBackupControlFile(const char *path) {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, + {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, {'u', 0, "checksum_version", &backup->checksum_version, SOURCE_FILE_STRICT}, {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, {'s', 0, "status", &status, SOURCE_FILE_STRICT}, {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, + {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &from_replica, SOURCE_FILE_STRICT}, {0} }; @@ -615,14 +626,32 @@ pgBackupCompareIdDesc(const void *l, const void *r) */ void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) +{ + pgBackupGetPath2(backup, path, len, subdir, NULL); +} + +/* + * Construct absolute path of the backup directory. + * Append "subdir1" and "subdir2" to the backup directory. + */ +void +pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2) { char *datetime; datetime = base36enc(backup->start_time); - if (subdir) - snprintf(path, len, "%s/%s/%s/%s", backup_path, BACKUPS_DIR, datetime, subdir); + + /* If "subdir1" is NULL do not check "subdir2" */ + if (!subdir1) + snprintf(path, len, "%s/%s", backup_instance_path, datetime); + else if (!subdir2) + snprintf(path, len, "%s/%s/%s", backup_instance_path, datetime, subdir1); + /* "subdir1" and "subdir2" is not NULL */ else - snprintf(path, len, "%s/%s/%s", backup_path, BACKUPS_DIR, datetime); + snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, + datetime, subdir1, subdir2); + free(datetime); make_native_path(path); diff --git a/src/configure.c b/src/configure.c new file mode 100644 index 00000000..8d131c72 --- /dev/null +++ b/src/configure.c @@ -0,0 +1,240 @@ +/*------------------------------------------------------------------------- + * + * configure.c: - manage backup catalog. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +static void opt_log_level(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); + +static pgBackupConfig *cur_config = NULL; + +/* Set configure options */ +int +do_configure(bool show_only) +{ + pgBackupConfig *config = readBackupCatalogConfigFile(); + if (pgdata) + config->pgdata = pgdata; + if (pgut_dbname) + config->pgdatabase = pgut_dbname; + if (host) + config->pghost = host; + if (port) + config->pgport = port; + if (username) + config->pguser = username; + + if (master_host) + config->master_host = master_host; + if (master_port) + config->master_port = master_port; + if (master_db) + config->master_db = master_db; + if (master_user) + config->master_user = master_user; + if (replica_timeout != 300) /* 300 is default value */ + config->replica_timeout = replica_timeout; + + if (log_level_defined) + config->log_level = log_level; + if (log_filename) + config->log_filename = log_filename; + if (error_log_filename) + config->error_log_filename = error_log_filename; + if (log_directory) + config->log_directory = log_directory; + if (log_rotation_size) + config->log_rotation_size = log_rotation_size; + if (log_rotation_age) + config->log_rotation_age = log_rotation_age; + + if (retention_redundancy) + config->retention_redundancy = retention_redundancy; + if (retention_window) + config->retention_window = retention_window; + + if (compress_alg != NOT_DEFINED_COMPRESS) + config->compress_alg = compress_alg; + if (compress_level != DEFAULT_COMPRESS_LEVEL) + config->compress_level = compress_level; + + if (show_only) + writeBackupCatalogConfig(stderr, config); + else + writeBackupCatalogConfigFile(config); + + return 0; +} + +void +pgBackupConfigInit(pgBackupConfig *config) +{ + config->system_identifier = 0; + config->pgdata = NULL; + config->pgdatabase = NULL; + config->pghost = NULL; + config->pgport = NULL; + config->pguser = NULL; + + config->master_host = NULL; + config->master_port = NULL; + config->master_db = NULL; + config->master_user = NULL; + config->replica_timeout = INT_MIN; /* INT_MIN means "undefined" */ + + config->log_level = INT_MIN; /* INT_MIN means "undefined" */ + config->log_filename = NULL; + config->error_log_filename = NULL; + config->log_directory = NULL; + config->log_rotation_size = 0; + config->log_rotation_age = 0; + + config->retention_redundancy = 0; + config->retention_window = 0; + + config->compress_alg = NOT_DEFINED_COMPRESS; + config->compress_level = DEFAULT_COMPRESS_LEVEL; +} + +void +writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) +{ + fprintf(out, "#Backup instance info\n"); + fprintf(out, "PGDATA = %s\n", config->pgdata); + fprintf(out, "system-identifier = %li\n", config->system_identifier); + + fprintf(out, "#Connection parameters:\n"); + if (config->pgdatabase) + fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); + if (config->pghost) + fprintf(out, "PGHOST = %s\n", config->pghost); + if (config->pgport) + fprintf(out, "PGPORT = %s\n", config->pgport); + if (config->pguser) + fprintf(out, "PGUSER = %s\n", config->pguser); + + fprintf(out, "#Replica parameters:\n"); + if (config->master_host) + fprintf(out, "master-host = %s\n", config->master_host); + if (config->master_port) + fprintf(out, "master-port = %s\n", config->master_port); + if (config->master_db) + fprintf(out, "master-db = %s\n", config->master_db); + if (config->master_user) + fprintf(out, "master-user = %s\n", config->master_user); + if (config->replica_timeout != INT_MIN) + fprintf(out, "replica_timeout = %d\n", config->replica_timeout); + + fprintf(out, "#Logging parameters:\n"); + if (config->log_level != INT_MIN) + fprintf(out, "log-level = %s\n", deparse_log_level(config->log_level)); + if (config->log_filename) + fprintf(out, "log-filename = %s\n", config->log_filename); + if (config->error_log_filename) + fprintf(out, "error-log-filename = %s\n", config->error_log_filename); + if (config->log_directory) + fprintf(out, "log-directory = %s\n", config->log_directory); + if (config->log_rotation_size) + fprintf(out, "log-rotation-size = %d\n", config->log_rotation_size); + if (config->log_rotation_age) + fprintf(out, "log-rotation-age = %d\n", config->log_rotation_age); + + fprintf(out, "#Retention parameters:\n"); + if (config->retention_redundancy) + fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); + if (config->retention_window) + fprintf(out, "retention-window = %u\n", config->retention_window); + + fprintf(out, "#Compression parameters:\n"); + + fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); + + if (compress_level != config->compress_level) + fprintf(out, "compress-level = %d\n", compress_level); + else + fprintf(out, "compress-level = %d\n", config->compress_level); +} + +void +writeBackupCatalogConfigFile(pgBackupConfig *config) +{ + char path[MAXPGPATH]; + FILE *fp; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot create %s: %s", + BACKUP_CATALOG_CONF_FILE, strerror(errno)); + + writeBackupCatalogConfig(fp, config); + fclose(fp); +} + + +pgBackupConfig* +readBackupCatalogConfigFile(void) +{ + pgBackupConfig *config = pgut_new(pgBackupConfig); + char path[MAXPGPATH]; + + pgut_option options[] = + { + /* retention options */ + { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, + { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, + /* compression options */ + { 'f', 36, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 37, "compress-level", &(config->compress_level), SOURCE_CMDLINE }, + /* logging options */ + { 'f', 40, "log-level", opt_log_level, SOURCE_CMDLINE }, + { 's', 41, "log-filename", &(config->log_filename), SOURCE_CMDLINE }, + { 's', 42, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, + { 's', 43, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, + { 'u', 44, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE }, + { 'u', 45, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE }, + /* connection options */ + { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, + { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, + { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, + { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, + { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, + /* replica options */ + { 's', 0, "master-host", &(config->master_host), SOURCE_FILE_STRICT }, + { 's', 0, "master-port", &(config->master_port), SOURCE_FILE_STRICT }, + { 's', 0, "master-db", &(config->master_db), SOURCE_FILE_STRICT }, + { 's', 0, "master-user", &(config->master_user), SOURCE_FILE_STRICT }, + { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE }, + /* other options */ + { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, + {0} + }; + + cur_config = config; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + + pgBackupConfigInit(config); + pgut_readopt(path, options, ERROR); + + return config; + +} + +static void +opt_log_level(pgut_option *opt, const char *arg) +{ + cur_config->log_level = parse_log_level(arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + cur_config->compress_alg = parse_compress_alg(arg); +} diff --git a/data.c b/src/data.c similarity index 73% rename from data.c rename to src/data.c index c44873e0..bcf89d94 100644 --- a/data.c +++ b/src/data.c @@ -19,10 +19,63 @@ #include "storage/block.h" #include "storage/bufpage.h" #include "storage/checksum_impl.h" +#include +#include + +static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size) +{ + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, compress_level); + return rc == Z_OK ? compressed_size : rc; +} + +static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size) +{ + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); + return rc == Z_OK ? dest_len : rc; +} + +static size_t +do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; + case ZLIB_COMPRESS: + return zlib_compress(dst, dst_size, src, src_size); + case PGLZ_COMPRESS: + return pglz_compress(src, src_size, dst, PGLZ_strategy_always); + } + + return -1; +} + +static size_t +do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; + case ZLIB_COMPRESS: + return zlib_decompress(dst, dst_size, src, src_size); + case PGLZ_COMPRESS: + return pglz_decompress(src, src_size, dst, dst_size); + } + + return -1; +} + + typedef struct BackupPageHeader { BlockNumber block; /* block number */ + int32 compressed_size; } BackupPageHeader; /* Verify page's header */ @@ -61,8 +114,10 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupPageHeader header; off_t offset; DataPage page; /* used as read buffer */ - size_t write_buffer_size = sizeof(header) + BLCKSZ; - char write_buffer[write_buffer_size]; + DataPage compressed_page; /* used as read buffer */ + size_t write_buffer_size; + /* maximum size of write buffer */ + char write_buffer[BLCKSZ+sizeof(header)]; size_t read_len = 0; XLogRecPtr page_lsn; int try_checksum = 100; @@ -92,12 +147,6 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, * If after several attempts page header is still invalid, throw an error. * The same idea is applied to checksum verification. */ - - /* - * TODO Should we show a hint about possible false positives suggesting to - * decrease concurrent load? Or we can just copy this page and rely on - * xlog recovery, marking backup as untrusted. - */ if (!parse_page(&page, &page_lsn)) { int i; @@ -119,9 +168,8 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, /* Try to read and verify this page again several times. */ if (try_checksum) { - if (verbose) - elog(WARNING, "File: %s blknum %u have wrong page header, try again", - file->path, blknum); + elog(WARNING, "File: %s blknum %u have wrong page header, try again", + file->path, blknum); usleep(100); continue; } @@ -150,9 +198,8 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, { if (try_checksum) { - if (verbose) - elog(WARNING, "File: %s blknum %u have wrong checksum, try again", - file->path, blknum); + elog(WARNING, "File: %s blknum %u have wrong checksum, try again", + file->path, blknum); usleep(100); } else @@ -164,9 +211,31 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, file->read_size += read_len; - memcpy(write_buffer, &header, sizeof(header)); - /* TODO implement block compression here? */ - memcpy(write_buffer + sizeof(header), page.data, BLCKSZ); + header.compressed_size = do_compress(compressed_page.data, sizeof(compressed_page.data), + page.data, sizeof(page.data), compress_alg); + + file->compress_alg = compress_alg; + + Assert (header.compressed_size <= BLCKSZ); + write_buffer_size = sizeof(header); + + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), compressed_page.data, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page.data, BLCKSZ); + write_buffer_size += header.compressed_size; + } + + /* Update CRC */ + COMP_CRC32C(*crc, &write_buffer, write_buffer_size); + /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) { @@ -177,10 +246,6 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, file->path, blknum, strerror(errno_tmp)); } - /* update CRC */ - COMP_CRC32C(*crc, &header, sizeof(header)); - COMP_CRC32C(*crc, page.data, BLCKSZ); - file->write_size += write_buffer_size; } @@ -188,6 +253,9 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, * Backup data file in the from_root directory to the to_root directory with * same relative path. If prev_backup_start_lsn is not NULL, only pages with * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental backup), validate checksum, optionally compress and write to + * backup with special header. */ bool backup_data_file(const char *from_root, const char *to_root, @@ -275,6 +343,7 @@ backup_data_file(const char *from_root, const char *to_root, n_blocks_read++; } + pg_free(file->pagemap.bitmap); pg_free(iter); } @@ -293,15 +362,11 @@ backup_data_file(const char *from_root, const char *to_root, FIN_CRC32C(file->crc); - /* Treat empty file as not-datafile. TODO Why? */ - if (file->read_size == 0) - file->is_datafile = false; - /* * If we have pagemap then file can't be a zero size. * Otherwise, we will clear the last file. */ - if (n_blocks_read == n_blocks_skipped) + if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) { if (remove(to_path) == -1) elog(ERROR, "cannot remove file \"%s\": %s", to_path, @@ -314,7 +379,6 @@ backup_data_file(const char *from_root, const char *to_root, /* * Restore compressed file that was backed up partly. - * TODO review */ static void restore_file_partly(const char *from_root,const char *to_root, pgFile *file) @@ -394,8 +458,6 @@ restore_file_partly(const char *from_root,const char *to_root, pgFile *file) write_size += read_len; } -// elog(LOG, "restore_file_partly(). %s write_size %lu, file->write_size %lu", -// file->path, write_size, file->write_size); /* update file permission */ if (chmod(to_path, file->mode) == -1) @@ -417,13 +479,10 @@ restore_compressed_file(const char *from_root, const char *to_root, pgFile *file) { - if (file->is_partial_copy == 0) + if (!file->is_partial_copy) copy_file(from_root, to_root, file); - else if (file->is_partial_copy == 1) - restore_file_partly(from_root, to_root, file); else - elog(ERROR, "restore_compressed_file(). Unknown is_partial_copy value %d", - file->is_partial_copy); + restore_file_partly(from_root, to_root, file); } /* @@ -470,7 +529,8 @@ restore_data_file(const char *from_root, for (blknum = 0; ; blknum++) { size_t read_len; - DataPage page; /* used as read buffer */ + DataPage compressed_page; /* used as read buffer */ + DataPage page; /* read BackupPageHeader */ read_len = fread(&header, 1, sizeof(header), in); @@ -484,48 +544,54 @@ restore_data_file(const char *from_root, "odd size page found at block %u of \"%s\"", blknum, file->path); else - elog(ERROR, "cannot read block %u of \"%s\": %s", + elog(ERROR, "cannot read header of block %u of \"%s\": %s", blknum, file->path, strerror(errno_tmp)); } if (header.block < blknum) - elog(ERROR, "backup is broken at block %u", - blknum); + elog(ERROR, "backup is broken at block %u", blknum); + Assert(header.compressed_size <= BLCKSZ); - if (fread(page.data, 1, BLCKSZ, in) != BLCKSZ) - elog(ERROR, "cannot read block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + read_len = fread(compressed_page.data, 1, + MAXALIGN(header.compressed_size), in); + if (read_len != MAXALIGN(header.compressed_size)) + elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + blknum, file->path, read_len, header.compressed_size); - /* update checksum because we are not save whole */ - if(backup->checksum_version) + if (header.compressed_size < BLCKSZ) { - bool is_zero_page = false; + size_t uncompressed_size = 0; - if(page.page_data.pd_upper == 0) - { - int i; - for(i = 0; i < BLCKSZ && page.data[i] == 0; i++); - if (i == BLCKSZ) - is_zero_page = true; - } + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + header.compressed_size, file->compress_alg); - /* skip calc checksum if zero page */ - if (!is_zero_page) - ((PageHeader) page.data)->pd_checksum = pg_checksum_page(page.data, file->segno * RELSEG_SIZE + header.block); + if (uncompressed_size != BLCKSZ) + elog(ERROR, "page uncompressed to %ld bytes. != BLCKSZ", uncompressed_size); } /* - * Seek and write the restored page. Backup might have holes in - * differential backups. + * Seek and write the restored page. */ blknum = header.block; if (fseek(out, blknum * BLCKSZ, SEEK_SET) < 0) elog(ERROR, "cannot seek block %u of \"%s\": %s", blknum, to_path, strerror(errno)); - if (fwrite(page.data, 1, sizeof(page), out) != sizeof(page)) - elog(ERROR, "cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + + if (header.compressed_size < BLCKSZ) + { + if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + else + { + /* if page wasn't compressed, we've read full block */ + if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } } /* update file permission */ @@ -543,20 +609,10 @@ restore_data_file(const char *from_root, fclose(out); } -/* If someone's want to use this function before correct - * generation values is set, he can look up for corresponding - * .cfm file in the file_list - */ -bool -is_compressed_data_file(pgFile *file) -{ - return (file->generation != -1); -} - /* - * Add check that file is not bigger than RELSEG_SIZE. - * WARNING compressed file can be exceed this limit. - * Add compression. + * Copy file to backup. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. */ bool copy_file(const char *from_root, const char *to_root, pgFile *file) @@ -614,6 +670,8 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* copy content and calc CRC */ for (;;) { + read_len = 0; + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) break; @@ -629,8 +687,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* update CRC */ COMP_CRC32C(crc, buf, read_len); - file->write_size += sizeof(buf); - file->read_size += sizeof(buf); + file->read_size += read_len; } errno_tmp = errno; @@ -657,10 +714,10 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* update CRC */ COMP_CRC32C(crc, buf, read_len); - file->write_size += read_len; file->read_size += read_len; } + file->write_size = file->read_size; /* finish CRC calculation and store into pgFile */ FIN_CRC32C(crc); file->crc = crc; @@ -681,6 +738,105 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) return true; } +/* Almost like copy file, except the fact we don't calculate checksum */ +void +copy_wal_file(const char *from_path, const char *to_path) +{ + FILE *in; + FILE *out; + size_t read_len = 0; + int errno_tmp; + char buf[XLOG_BLCKSZ]; + struct stat st; + + /* open file for read */ + in = fopen(from_path, "r"); + if (in == NULL) + { + /* maybe deleted, it's not error */ + if (errno == ENOENT) + elog(ERROR, "cannot open source WAL file \"%s\": %s", from_path, + strerror(errno)); + } + + /* open backup file for write */ + out = fopen(to_path, "w"); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot stat \"%s\": %s", from_path, + strerror(errno)); + } + + if (st.st_size > XLOG_SEG_SIZE) + elog(ERROR, "Unexpected wal file size %s : %ld", from_path, st.st_size); + + /* copy content */ + for (;;) + { + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + break; + + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + from_path, strerror(errno_tmp)); + } + + /* copy odd part */ + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + } + + + /* update file permission. */ + if (chmod(to_path, st.st_mode) == -1) + { + errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + fclose(in); + fclose(out); + +} + /* * Save part of the file into backup. * skip_size - size of the file in previous backup. We can skip it @@ -799,9 +955,7 @@ copy_file_partly(const char *from_root, const char *to_root, } /* add meta information needed for recovery */ - file->is_partial_copy = 1; - -// elog(LOG, "copy_file_partly(). %s file->write_size %lu", to_path, file->write_size); + file->is_partial_copy = true; fclose(in); fclose(out); diff --git a/delete.c b/src/delete.c similarity index 84% rename from delete.c rename to src/delete.c index 55118fc3..f33dd3a5 100644 --- a/delete.c +++ b/src/delete.c @@ -15,8 +15,7 @@ #include static int pgBackupDeleteFiles(pgBackup *backup); -static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, - bool delete_all); +static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli); int do_delete(time_t backup_id) @@ -108,7 +107,7 @@ do_delete(time_t backup_id) } } - delete_walfiles(oldest_lsn, oldest_tli, true); + delete_walfiles(oldest_lsn, oldest_tli); } /* cleanup */ @@ -131,7 +130,7 @@ do_retention_purge(void) size_t i; time_t days_threshold = time(NULL) - (retention_window * 60 * 60 * 24); XLogRecPtr oldest_lsn = InvalidXLogRecPtr; - TimeLineID oldest_tli; + TimeLineID oldest_tli = 0; bool keep_next_backup = true; /* Do not delete first full backup */ bool backup_deleted = false; /* At least one backup was deleted */ @@ -200,7 +199,7 @@ do_retention_purge(void) } /* Purge WAL files */ - delete_walfiles(oldest_lsn, oldest_tli, true); + delete_walfiles(oldest_lsn, oldest_tli); /* Cleanup */ parray_walk(backup_list, pgBackupFree); @@ -280,13 +279,16 @@ pgBackupDeleteFiles(pgBackup *backup) } /* - * Delete WAL segments up to oldest_lsn. + * Deletes WAL segments up to oldest_lsn or all WAL segments (if all backups + * was deleted and so oldest_lsn is invalid). * - * If oldest_lsn is invalid function exists. But if delete_all is true then - * WAL segements will be deleted anyway. + * oldest_lsn - if valid, function deletes WAL segments, which contain lsn + * older than oldest_lsn. If it is invalid function deletes all WAL segments. + * oldest_tli - is used to construct oldest WAL segment in addition to + * oldest_lsn. */ static void -delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) +delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli) { XLogSegNo targetSegNo; char oldestSegmentNeeded[MAXFNAMELEN]; @@ -297,9 +299,6 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) char min_wal_file[MAXPGPATH]; int rc; - if (XLogRecPtrIsInvalid(oldest_lsn) && !delete_all) - return; - max_wal_file[0] = '\0'; min_wal_file[0] = '\0'; @@ -356,8 +355,7 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) wal_file, strerror(errno)); break; } - if (verbose) - elog(LOG, "removed WAL segment \"%s\"", wal_file); + elog(LOG, "removed WAL segment \"%s\"", wal_file); if (max_wal_file[0] == '\0' || strcmp(max_wal_file + 8, arcde->d_name + 8) < 0) @@ -370,9 +368,9 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) } } - if (!verbose && min_wal_file[0] != '\0') + if (min_wal_file[0] != '\0') elog(INFO, "removed min WAL segment \"%s\"", min_wal_file); - if (!verbose && max_wal_file[0] != '\0') + if (max_wal_file[0] != '\0') elog(INFO, "removed max WAL segment \"%s\"", max_wal_file); if (errno) @@ -386,3 +384,48 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) elog(WARNING, "could not open archive location \"%s\": %s", arclog_path, strerror(errno)); } + + +/* Delete all backup files and wal files of given instance. */ +int +do_delete_instance(void) +{ + parray *backup_list; + int i; + char instance_config_path[MAXPGPATH]; + + /* Delete all backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + pgBackupDeleteFiles(backup); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + /* Delete all wal files. */ + delete_walfiles(InvalidXLogRecPtr, 0); + + /* Delete backup instance config file */ + join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + if (remove(instance_config_path)) + { + elog(ERROR, "can't remove \"%s\": %s", instance_config_path, + strerror(errno)); + } + + /* Delete instance root directories */ + if (rmdir(backup_instance_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + if (rmdir(arclog_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + + elog(INFO, "Instance '%s' successfully deleted", instance_name); + return 0; +} \ No newline at end of file diff --git a/dir.c b/src/dir.c similarity index 95% rename from dir.c rename to src/dir.c index 720cf2e1..5aed3acd 100644 --- a/dir.c +++ b/src/dir.c @@ -154,8 +154,10 @@ pgFileInit(const char *path) file->segno = 0; file->path = pgut_malloc(strlen(path) + 1); strcpy(file->path, path); /* enough buffer size guaranteed */ - file->generation = -1; - file->is_partial_copy = 0; + file->is_cfs = false; + file->generation = 0; + file->is_partial_copy = false; + file->compress_alg = NOT_DEFINED_COMPRESS; return file; } @@ -307,7 +309,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray *black_list = NULL; char path[MAXPGPATH]; - join_path_components(path, backup_path, PG_BLACK_LIST); + join_path_components(path, backup_instance_path, PG_BLACK_LIST); /* List files with black list */ if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) { @@ -344,7 +346,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, } /* - * TODO Add comment, review + * TODO Add comment */ static void dir_list_file_internal(parray *files, const char *root, bool exclude, @@ -633,7 +635,7 @@ read_tablespace_map(parray *files, const char *backup_dir) path[MAXPGPATH]; pgFile *file; - if (sscanf(buf, "%s %s", link_name, path) != 2) + if (sscanf(buf, "%1023s %1023s", link_name, path) != 2) elog(ERROR, "invalid format found in \"%s\"", map_path); file = pgut_new(pgFile); @@ -671,20 +673,21 @@ print_file_list(FILE *out, const parray *files, const char *root) path = GetRelativePath(path, root); fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\"," - "\"is_datafile\":\"%u\", \"crc\":\"%u\"", + "\"is_datafile\":\"%u\", \"crc\":\"%u\", \"compress_alg\":\"%s\"", path, (unsigned long) file->write_size, file->mode, - file->is_datafile?1:0, file->crc); + file->is_datafile?1:0, file->crc, deparse_compress_alg(file->compress_alg)); if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); - /* TODO What for do we write it to file? */ if (S_ISLNK(file->mode)) fprintf(out, ",\"linked\":\"%s\"", file->linked); #ifdef PGPRO_EE - fprintf(out, ",\"CFS_generation\":\"" UINT64_FORMAT "\",\"is_partial_copy\":\"%d\"", - file->generation, file->is_partial_copy); + if (file->is_cfs) + fprintf(out, ",\"is_cfs\":\"%u\" ,\"CFS_generation\":\"" UINT64_FORMAT "\"," + "\"is_partial_copy\":\"%u\"", + file->is_cfs?1:0, file->generation, file->is_partial_copy?1:0); #endif fprintf(out, "}\n"); } @@ -847,6 +850,7 @@ dir_read_file_list(const char *root, const char *file_txt) char path[MAXPGPATH]; char filepath[MAXPGPATH]; char linked[MAXPGPATH]; + char compress_alg_string[MAXPGPATH]; uint64 write_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, @@ -854,7 +858,8 @@ dir_read_file_list(const char *root, const char *file_txt) segno; #ifdef PGPRO_EE uint64 generation, - is_partial_copy; + is_partial_copy, + is_cfs; #endif pgFile *file; @@ -867,10 +872,12 @@ dir_read_file_list(const char *root, const char *file_txt) /* optional fields */ get_control_value(buf, "linked", linked, NULL, false); get_control_value(buf, "segno", NULL, &segno, false); + get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); #ifdef PGPRO_EE - get_control_value(buf, "CFS_generation", NULL, &generation, true); - get_control_value(buf, "is_partial_copy", NULL, &is_partial_copy, true); + get_control_value(buf, "is_cfs", NULL, &is_cfs, false); + get_control_value(buf, "CFS_generation", NULL, &generation, false); + get_control_value(buf, "is_partial_copy", NULL, &is_partial_copy, false); #endif if (root) join_path_components(filepath, root, path); @@ -883,12 +890,14 @@ dir_read_file_list(const char *root, const char *file_txt) file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->crc = (pg_crc32) crc; + file->compress_alg = parse_compress_alg(compress_alg_string); if (linked[0]) file->linked = pgut_strdup(linked); file->segno = (int) segno; #ifdef PGPRO_EE + file->is_cfs = is_cfs ? true : false; file->generation = generation; - file->is_partial_copy = (int) is_partial_copy; + file->is_partial_copy = is_partial_copy ? true : false; #endif parray_append(files, file); diff --git a/fetch.c b/src/fetch.c similarity index 100% rename from fetch.c rename to src/fetch.c diff --git a/src/help.c b/src/help.c new file mode 100644 index 00000000..1f116528 --- /dev/null +++ b/src/help.c @@ -0,0 +1,339 @@ +/*------------------------------------------------------------------------- + * + * help.c + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +static void help_init(void); +static void help_backup(void); +static void help_restore(void); +static void help_validate(void); +static void help_show(void); +static void help_delete(void); +static void help_set_config(void); +static void help_show_config(void); +static void help_add_instance(void); +static void help_del_instance(void); + +void +help_command(char *command) +{ + if (strcmp(command, "init") == 0) + help_init(); + else if (strcmp(command, "backup") == 0) + help_backup(); + else if (strcmp(command, "restore") == 0) + help_restore(); + else if (strcmp(command, "validate") == 0) + help_validate(); + else if (strcmp(command, "show") == 0) + help_show(); + else if (strcmp(command, "delete") == 0) + help_delete(); + else if (strcmp(command, "set-config") == 0) + help_set_config(); + else if (strcmp(command, "show-config") == 0) + help_show_config(); + else if (strcmp(command, "add-instance") == 0) + help_add_instance(); + else if (strcmp(command, "del-instance") == 0) + help_del_instance(); + else if (strcmp(command, "--help") == 0 + || strcmp(command, "help") == 0 + || strcmp(command, "-?") == 0 + || strcmp(command, "--version") == 0 + || strcmp(command, "version") == 0 + || strcmp(command, "-V") == 0) + printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); + else + printf(_("Unknown command. Try pg_probackup help\n")); + exit(0); +} + +void +help_pg_probackup(void) +{ + printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + + printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); + + printf(_("\n %s version\n"), PROGRAM_NAME); + + printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); + + printf(_("\n %s set-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level=log-level]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_("\n %s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + + printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--progress] [--delete-expired]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_("\n %s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + + printf(_("\n %s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline]\n")); + + printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + + printf(_("\n %s delete -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--wal] [-i backup-id | --expired]\n")); + + printf(_("\n %s add-instance -B backup-dir -D pgdata-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s del-instance -B backup-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + if ((PROGRAM_URL || PROGRAM_EMAIL)) + { + printf("\n"); + if (PROGRAM_URL) + printf("Read the website for details. <%s>\n", PROGRAM_URL); + if (PROGRAM_EMAIL) + printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); + } + exit(0); +} + +static void +help_init(void) +{ + printf(_("%s init -B backup-path -D pgdata-dir\n\n"), PROGRAM_NAME); + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); +} + +static void +help_backup(void) +{ + printf(_("%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress] [--delete-expired]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|PTRACK\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); + printf(_(" --stream stream the transaction log and include it in the backup\n")); + printf(_(" --archive-timeout wait timeout for WAL segment archiving\n")); + printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --backup-pg-log backup of pg_log directory\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + printf(_(" --delete-expired delete backups expired according to current\n")); + printf(_(" retention policy after successful backup completion\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9]\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -d, --dbname=DBNAME database to connect\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); + printf(_(" -p, --port=PORT database server port\n")); + printf(_(" -U, --username=USERNAME user name to connect as\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication in seconds\n")); +} + +static void +help_restore(void) +{ + printf(_("%s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); + printf(_(" -i, --backup-id=backup-id backup to restore\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); +} + +static void +help_validate(void) +{ + printf(_("%s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to validate\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); +} + +static void +help_show(void) +{ + printf(_("%s show -B backup-dir\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name show info about specific intstance\n")); + printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); +} + +static void +help_delete(void) +{ + printf(_("%s delete -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--wal] [-i backup-id | --expired]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --wal remove unnecessary wal files\n")); + printf(_(" -i, --backup-id=backup-id backup to delete\n")); + printf(_(" --expired delete backups expired according to current\n")); + printf(_(" retention policy\n")); +} + +static void +help_set_config(void) +{ + printf(_("%s set-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level=log-level]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level=log-level controls which message levels are sent to the log\n")); + printf(_(" --log-filename=log-filename file names of the created log files which is treated as as strftime pattern\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" file names of the created log files for error messages\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory in which log files will be created\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" maximum size of an individual log file in kilobytes\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" maximum lifetime of an individual log file in minutes\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9]\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -d, --dbname=DBNAME database to connect\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); + printf(_(" -p, --port=PORT database server port\n")); + printf(_(" -U, --username=USERNAME user name to connect as\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name=host_name database server host of master\n")); + printf(_(" --master-port=port=port database server port of master\n")); + printf(_(" --master-user=user_name=user_name user name to connect to master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication\n")); +} + +static void +help_show_config(void) +{ + printf(_("%s show-config -B backup-dir --instance=instance_name\n\n"), PROGRAM_NAME); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); +} + +static void +help_add_instance(void) +{ + printf(_("%s add-instance -B backup-dir -D pgdata-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); + printf(_(" --instance=instance_name name of the new instance\n")); +} + +static void +help_del_instance(void) +{ + printf(_("%s del-instance -B backup-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); +} diff --git a/init.c b/src/init.c similarity index 53% rename from init.c rename to src/init.c index 9a85022d..d92704cb 100644 --- a/init.c +++ b/src/init.c @@ -12,6 +12,7 @@ #include #include +#include /* * selects function for scandir. @@ -30,15 +31,8 @@ do_init(void) { char path[MAXPGPATH]; char arclog_path_dir[MAXPGPATH]; - struct dirent **dp; int results; - pgBackupConfig *config = pgut_new(pgBackupConfig); - - /* PGDATA is always required */ - if (pgdata == NULL) - elog(ERROR, "Required parameter not specified: PGDATA " - "(-D, --pgdata)"); if (access(backup_path, F_OK) == 0) { @@ -47,20 +41,63 @@ do_init(void) elog(ERROR, "backup catalog already exist and it's not empty"); } - /* Read system_identifier from PGDATA */ - system_identifier = get_system_identifier(); - /* create backup catalog root directory */ dir_create_dir(backup_path, DIR_PERMISSION); - /* create directories for backup of online files */ + /* create backup catalog data directory */ join_path_components(path, backup_path, BACKUPS_DIR); dir_create_dir(path, DIR_PERMISSION); - /* Create "wal" directory */ + /* create backup catalog wal directory */ join_path_components(arclog_path_dir, backup_path, "wal"); dir_create_dir(arclog_path_dir, DIR_PERMISSION); + elog(INFO, "Backup catalog '%s' successfully inited", backup_path); + return 0; +} + +int +do_add_instance(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + struct stat st; + pgBackupConfig *config = pgut_new(pgBackupConfig); + + /* PGDATA is always required */ + if (pgdata == NULL) + elog(ERROR, "Required parameter not specified: PGDATA " + "(-D, --pgdata)"); + + /* Read system_identifier from PGDATA */ + system_identifier = get_system_identifier(pgdata); + + /* Ensure that all root directories already exist */ + if (access(backup_path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", backup_path); + + join_path_components(path, backup_path, BACKUPS_DIR); + if (access(path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", path); + + join_path_components(arclog_path_dir, backup_path, "wal"); + if (access(arclog_path_dir, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", arclog_path_dir); + + /* Create directory for data files of this specific instance */ + if (stat(backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "instance '%s' already exists", backup_instance_path); + dir_create_dir(backup_instance_path, DIR_PERMISSION); + + /* + * Create directory for wal files of this specific instance. + * Existence check is extra paranoid because if we don't have such a + * directory in data dir, we shouldn't have it in wal as well. + */ + if (stat(arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "arclog_path '%s' already exists", arclog_path); + dir_create_dir(arclog_path, DIR_PERMISSION); + /* * Wite initial config. system-identifier and pgdata are set in * init subcommand and will never be updated. @@ -70,5 +107,6 @@ do_init(void) config->pgdata = pgdata; writeBackupCatalogConfigFile(config); + elog(INFO, "Instance '%s' successfully inited", instance_name); return 0; } diff --git a/parsexlog.c b/src/parsexlog.c similarity index 69% rename from parsexlog.c rename to src/parsexlog.c index f94baaea..aa8de7ed 100644 --- a/parsexlog.c +++ b/src/parsexlog.c @@ -110,6 +110,14 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, XLogSegNo endSegNo, nextSegNo = 0; + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + private.archivedir = archivedir; private.tli = tli; xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); @@ -123,6 +131,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, do { record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) { XLogRecPtr errptr; @@ -130,12 +139,23 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, errptr = startpoint ? startpoint : xlogreader->EndRecPtr; if (errormsg) - elog(ERROR, "could not read WAL record at %X/%X: %s", + elog(WARNING, "could not read WAL record at %X/%X: %s", (uint32) (errptr >> 32), (uint32) (errptr), errormsg); else - elog(ERROR, "could not read WAL record at %X/%X", + elog(WARNING, "could not read WAL record at %X/%X", (uint32) (errptr >> 32), (uint32) (errptr)); + + /* + * If we don't have all WAL files from prev backup start_lsn to current + * start_lsn, we won't be able to build page map and PAGE backup will + * be incorrect. Stop it and throw an error. + */ + if (!xlogexists) + elog(ERROR, "WAL segment \"%s\" is absent", xlogfpath); + else if (xlogreadfd != -1) + elog(ERROR, "Possible WAL CORRUPTION." + "Error has occured during reading WAL segment \"%s\"", xlogfpath); } extractPageInfo(xlogreader); @@ -143,7 +163,6 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, startpoint = InvalidXLogRecPtr; /* continue reading at next record */ XLByteToSeg(xlogreader->EndRecPtr, nextSegNo); - } while (nextSegNo <= endSegNo && xlogreader->EndRecPtr != endpoint); XLogReaderFree(xlogreader); @@ -155,6 +174,98 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, } } +/* + * Ensure that the backup has all wal files needed for recovery to consistent state. + */ +static void +validate_backup_wal_from_start_to_stop(pgBackup *backup, + char *backup_xlog_path, + TimeLineID tli) +{ + XLogRecPtr startpoint = backup->start_lsn; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + bool got_endpoint = false; + + private.archivedir = backup_xlog_path; + private.tli = tli; + + /* We will check it in the end */ + xlogfpath[0] = '\0'; + + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + + while (true) + { + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + /* Got WAL record at stop_lsn */ + if (xlogreader->ReadRecPtr == backup->stop_lsn) + { + got_endpoint = true; + break; + } + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (!got_endpoint) + { + if (xlogfpath[0] != 0) + { + /* XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status below. + */ + if (!xlogexists) + { + elog(WARNING, "WAL segment \"%s\" is absent", xlogfpath); + } + else if (xlogreadfd != -1) + { + elog(WARNING, "Possible WAL CORRUPTION." + "Error has occured during reading WAL segment \"%s\"", xlogfpath); + } + } + + /* + * If we don't have WAL between start_lsn and stop_lsn, + * the backup is definitely corrupted. Update its status. + */ + backup->status = BACKUP_STATUS_CORRUPT; + pgBackupWriteBackupControlFile(backup); + elog(ERROR, "there are not enough WAL records to restore from %X/%X to %X/%X", + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn), + (uint32) (backup->stop_lsn >> 32), + (uint32) (backup->stop_lsn)); + } + + /* clean */ + XLogReaderFree(xlogreader); + if (xlogreadfd != -1) + { + close(xlogreadfd); + xlogreadfd = -1; + xlogexists = false; + } +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent + * state. And check if we have in archive all files needed to restore the backup + * up to the given recovery target. + */ void validate_wal(pgBackup *backup, const char *archivedir, @@ -163,6 +274,7 @@ validate_wal(pgBackup *backup, TimeLineID tli) { XLogRecPtr startpoint = backup->start_lsn; + char *backup_id; XLogRecord *record; XLogReaderState *xlogreader; char *errormsg; @@ -171,10 +283,63 @@ validate_wal(pgBackup *backup, TimestampTz last_time = 0; char last_timestamp[100], target_timestamp[100]; - bool all_wal = false, - got_endpoint = false; + bool all_wal = false; + char backup_xlog_path[MAXPGPATH]; + /* We need free() this later */ + backup_id = base36enc(backup->start_time); + + if (!XRecOffIsValid(backup->start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", + (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), + backup_id); + + if (!XRecOffIsValid(backup->stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", + (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), + backup_id); + + /* + * Check that the backup has all wal files needed + * for recovery to consistent state. + */ + if (backup->stream) + { + sprintf(backup_xlog_path, "/%s/%s/%s/%s", + backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); + + validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli); + } + else + validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli); + + free(backup_id); + + /* + * If recovery target is provided check that we can restore backup to a + * recoverty target time or xid. + */ + if (!TransactionIdIsValid(target_xid) || target_time == 0) + { + /* Recoverty target is not given so exit */ + elog(INFO, "backup validation completed successfully"); + return; + } + + /* + * If recovery target is provided, ensure that archive files exist in + * archive directory. + */ + if (dir_is_empty(archivedir)) + elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); + + /* + * Check if we have in archive all files needed to restore backup + * up to the given recovery target. + * In any case we cannot restore to the point before stop_lsn. + */ private.archivedir = archivedir; + private.tli = tli; xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); if (xlogreader == NULL) @@ -183,6 +348,15 @@ validate_wal(pgBackup *backup, /* We will check it in the end */ xlogfpath[0] = '\0'; + /* We can restore at least up to the backup end */ + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + last_xid = backup->recovery_xid; + + if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) + || (target_time != 0 && backup->recovery_time >= target_time)) + all_wal = true; + + startpoint = backup->stop_lsn; while (true) { bool timestamp_record; @@ -196,10 +370,6 @@ validate_wal(pgBackup *backup, break; } - /* Got WAL record at stop_lsn */ - if (xlogreader->ReadRecPtr == backup->stop_lsn) - got_endpoint = true; - timestamp_record = getRecordTimestamp(xlogreader, &last_time); if (XLogRecGetXid(xlogreader) != InvalidTransactionId) last_xid = XLogRecGetXid(xlogreader); @@ -230,54 +400,45 @@ validate_wal(pgBackup *backup, if (last_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), timestamptz_to_time_t(last_time)); - else - time2iso(last_timestamp, lengthof(last_timestamp), - backup->recovery_time); - if (last_xid == InvalidTransactionId) - last_xid = backup->recovery_xid; - /* There are all need WAL records */ + /* There are all needed WAL records */ if (all_wal) elog(INFO, "backup validation completed successfully on time %s and xid " XID_FMT, last_timestamp, last_xid); - /* There are not need WAL records */ + /* Some needed WAL records are absent */ else { if (xlogfpath[0] != 0) { - /* XLOG reader couldnt read WAL segment */ + /* XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status below. + */ if (!xlogexists) + { elog(WARNING, "WAL segment \"%s\" is absent", xlogfpath); + } else if (xlogreadfd != -1) - elog(ERROR, "Possible WAL CORRUPTION." - "Error has occured during reading WAL segment \"%s\"", xlogfpath); + { + elog(WARNING, "Possible WAL CORRUPTION." + "Error has occured during reading WAL segment \"%s\"", xlogfpath); + } } - if (!got_endpoint) - elog(ERROR, "there are not enough WAL records to restore from %X/%X to %X/%X", - (uint32) (backup->start_lsn >> 32), - (uint32) (backup->start_lsn), - (uint32) (backup->stop_lsn >> 32), - (uint32) (backup->stop_lsn)); - else - { - if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), - target_time); + elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, + last_timestamp, last_xid); - elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, - last_timestamp, last_xid); - - if (TransactionIdIsValid(target_xid) && target_time != 0) - elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, - target_timestamp, target_xid); - else if (TransactionIdIsValid(target_xid)) - elog(ERROR, "not enough WAL records to xid " XID_FMT, - target_xid); - else if (target_time != 0) - elog(ERROR, "not enough WAL records to time %s", - target_timestamp); - } + if (target_time > 0) + time2iso(target_timestamp, lengthof(target_timestamp), + target_time); + if (TransactionIdIsValid(target_xid) && target_time != 0) + elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, + target_timestamp, target_xid); + else if (TransactionIdIsValid(target_xid)) + elog(ERROR, "not enough WAL records to xid " XID_FMT, + target_xid); + else if (target_time != 0) + elog(ERROR, "not enough WAL records to time %s", + target_timestamp); } /* clean */ @@ -305,6 +466,14 @@ read_recovery_info(const char *archivedir, TimeLineID tli, XLogPageReadPrivate private; bool res; + if (!XRecOffIsValid(start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X", + (uint32) (start_lsn >> 32), (uint32) (start_lsn)); + + if (!XRecOffIsValid(stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + private.archivedir = archivedir; private.tli = tli; @@ -365,7 +534,8 @@ cleanup: } /* - * Check if WAL segment file 'wal_path' contains 'target_lsn'. + * Check if there is a WAL segment file in 'archivedir' which contains + * 'target_lsn'. */ bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, @@ -376,6 +546,10 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, char *errormsg; bool res; + if (!XRecOffIsValid(target_lsn)) + elog(ERROR, "Invalid target_lsn value %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + private.archivedir = archivedir; private.tli = target_tli; @@ -384,15 +558,7 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, elog(ERROR, "out of memory"); res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; - if (!res) - { - if (errormsg) - elog(ERROR, "could not read WAL record at %X/%X: %s", - (uint32) (target_lsn >> 32), (uint32) (target_lsn), - errormsg); - - /* Didn't find 'target_lsn' and there is no error, return false */ - } + /* Didn't find 'target_lsn' and there is no error, return false */ XLogReaderFree(xlogreader); if (xlogreadfd != -1) @@ -413,9 +579,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, { XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; uint32 targetPageOff; - XLogSegNo targetSegNo; - XLByteToSeg(targetPagePtr, targetSegNo); targetPageOff = targetPagePtr % XLogSegSize; /* @@ -478,8 +642,6 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, return -1; } - Assert(targetSegNo == xlogreadsegno); - *pageTLI = private->tli; return XLOG_BLCKSZ; } diff --git a/src/pg_probackup.c b/src/pg_probackup.c new file mode 100644 index 00000000..867df105 --- /dev/null +++ b/src/pg_probackup.c @@ -0,0 +1,475 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.c: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "streamutil.h" + +#include +#include +#include +#include +#include + +const char *PROGRAM_VERSION = "1.1.17"; +const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; +const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; + +/* directory options */ +char *backup_path = NULL; +char *pgdata = NULL; +/* + * path or to the data files in the backup catalog + * $BACKUP_PATH/backups/instance_name + */ +char backup_instance_path[MAXPGPATH]; +/* + * path or to the wal files in the backup catalog + * $BACKUP_PATH/wal/instance_name + */ +char arclog_path[MAXPGPATH] = ""; + +/* common options */ +char *backup_id_string_param = NULL; +int num_threads = 1; +bool stream_wal = false; +bool progress = false; + +/* backup options */ +bool backup_logs = false; +bool smooth_checkpoint; +bool from_replica = false; +/* Wait timeout for WAL segment archiving */ +uint32 archive_timeout = 300; /* default is 300 seconds */ +const char *master_db = NULL; +const char *master_host = NULL; +const char *master_port= NULL; +const char *master_user = NULL; +uint32 replica_timeout = 300; /* default is 300 seconds */ + +/* restore options */ +static char *target_time; +static char *target_xid; +static char *target_inclusive; +static TimeLineID target_tli; + +/* delete options */ +bool delete_wal = false; +bool delete_expired = false; +bool apply_to_all = false; +bool force_delete = false; + +/* retention options */ +uint32 retention_redundancy = 0; +uint32 retention_window = 0; + +/* compression options */ +CompressAlg compress_alg = NOT_DEFINED_COMPRESS; +int compress_level = DEFAULT_COMPRESS_LEVEL; + +/* other options */ +char *instance_name; +uint64 system_identifier = 0; + +/* archive push options */ +static char *wal_file_path; +static char *wal_file_name; + +/* current settings */ +pgBackup current; +ProbackupSubcmd backup_subcmd; + +bool help = false; + +static void opt_backup_mode(pgut_option *opt, const char *arg); +static void opt_log_level(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); + +static pgut_option options[] = +{ + /* directory options */ + { 'b', 1, "help", &help, SOURCE_CMDLINE }, + { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, + { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, + /* common options */ + { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, + { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, + { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, + { 's', 'i', "backup-id", &backup_id_string_param, SOURCE_CMDLINE }, + /* backup options */ + { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, + { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, + { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, + { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, + { 'u', 11, "archive-timeout", &archive_timeout, SOURCE_CMDLINE }, + { 'b', 12, "delete-expired", &delete_expired, SOURCE_CMDLINE }, + { 's', 13, "master-db", &master_db, SOURCE_CMDLINE, }, + { 's', 14, "master-host", &master_host, SOURCE_CMDLINE, }, + { 's', 15, "master-port", &master_port, SOURCE_CMDLINE, }, + { 's', 16, "master-user", &master_user, SOURCE_CMDLINE, }, + { 'u', 17, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, }, + /* restore options */ + { 's', 20, "time", &target_time, SOURCE_CMDLINE }, + { 's', 21, "xid", &target_xid, SOURCE_CMDLINE }, + { 's', 22, "inclusive", &target_inclusive, SOURCE_CMDLINE }, + { 'u', 23, "timeline", &target_tli, SOURCE_CMDLINE }, + { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, + /* delete options */ + { 'b', 30, "wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 31, "expired", &delete_expired, SOURCE_CMDLINE }, + { 'b', 32, "all", &apply_to_all, SOURCE_CMDLINE }, + /* TODO not implemented yet */ + { 'b', 33, "force", &force_delete, SOURCE_CMDLINE }, + /* retention options */ + { 'u', 34, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, + { 'u', 35, "retention-window", &retention_window, SOURCE_CMDLINE }, + /* compression options */ + { 'f', 36, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 37, "compress-level", &compress_level, SOURCE_CMDLINE }, + /* logging options */ + { 'f', 40, "log-level", opt_log_level, SOURCE_CMDLINE }, + { 's', 41, "log-filename", &log_filename, SOURCE_CMDLINE }, + { 's', 42, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, + { 's', 43, "log-directory", &log_directory, SOURCE_CMDLINE }, + { 'u', 44, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE }, + { 'u', 45, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE }, + /* connection options */ + { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, + { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, + { 's', 'p', "pgport", &port, SOURCE_CMDLINE }, + { 's', 'U', "pguser", &username, SOURCE_CMDLINE }, + { 'B', 'w', "no-password", &prompt_password, SOURCE_CMDLINE }, + /* other options */ + { 'U', 50, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, + { 's', 51, "instance", &instance_name, SOURCE_CMDLINE }, + /* archive-push options */ + { 's', 60, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, + { 's', 61, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, + { 0 } +}; + +/* + * Entry point of pg_probackup command. + */ +int +main(int argc, char *argv[]) +{ + char path[MAXPGPATH]; + /* Check if backup_path is directory. */ + struct stat stat_buf; + int rc; + + /* initialize configuration */ + pgBackup_init(¤t); + + PROGRAM_NAME = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], "pgscripts"); + + /* Parse subcommands and non-subcommand options */ + if (argc > 1) + { + if (strcmp(argv[1], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH; + else if (strcmp(argv[1], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET; + else if (strcmp(argv[1], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE; + else if (strcmp(argv[1], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE; + else if (strcmp(argv[1], "init") == 0) + backup_subcmd = INIT; + else if (strcmp(argv[1], "backup") == 0) + backup_subcmd = BACKUP; + else if (strcmp(argv[1], "restore") == 0) + backup_subcmd = RESTORE; + else if (strcmp(argv[1], "validate") == 0) + backup_subcmd = VALIDATE; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW; + else if (strcmp(argv[1], "delete") == 0) + backup_subcmd = DELETE; + else if (strcmp(argv[1], "set-config") == 0) + backup_subcmd = SET_CONFIG; + else if (strcmp(argv[1], "show-config") == 0) + backup_subcmd = SHOW_CONFIG; + else if (strcmp(argv[1], "--help") == 0 + || strcmp(argv[1], "help") == 0 + || strcmp(argv[1], "-?") == 0) + { + if (argc > 2) + help_command(argv[2]); + else + help_pg_probackup(); + } + else if (strcmp(argv[1], "--version") == 0 + || strcmp(argv[1], "version") == 0 + || strcmp(argv[1], "-V") == 0) + { + if (argc == 2) + { + fprintf(stderr, "%s %s\n", PROGRAM_NAME, PROGRAM_VERSION); + exit(0); + } + else if (strcmp(argv[2], "--help") == 0) + help_command(argv[1]); + else + elog(ERROR, "Invalid arguments for \"%s\" subcommand", argv[1]); + } + else + elog(ERROR, "Unknown subcommand"); + } + + /* Parse command line arguments */ + pgut_getopt(argc, argv, options); + + if (help) + help_command(argv[2]); + + /* backup_path is required for all pg_probackup commands except help */ + if (backup_path == NULL) + { + /* + * If command line argument is not set, try to read BACKUP_PATH + * from environment variable + */ + backup_path = getenv("BACKUP_PATH"); + if (backup_path == NULL) + elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + } + + /* Ensure that backup_path is an absolute path */ + if (!is_absolute_path(backup_path)) + elog(ERROR, "-B, --backup-path must be an absolute path"); + + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + + /* Option --instance is required for all commands except init and show */ + if (backup_subcmd != INIT && backup_subcmd != SHOW && backup_subcmd != VALIDATE) + { + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + } + + /* + * If --instance option was passed, construct paths for backup data and + * xlog files of this backup instance. + */ + if (instance_name) + { + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + + /* + * Ensure that requested backup instance exists. + * for all commands except init, which doesn't take this parameter + * and add-instance which creates new instance. + */ + if (backup_subcmd != INIT && backup_subcmd != ADD_INSTANCE) + { + if (access(backup_instance_path, F_OK) != 0) + elog(ERROR, "Instance '%s' does not exist in this backup catalog", + instance_name); + } + } + + /* + * Read options from env variables or from config file, + * unless we're going to set them via set-config. + */ + if (instance_name && backup_subcmd != SET_CONFIG) + { + /* Read environment variables */ + pgut_getopt_env(options); + + /* Read options from configuration file */ + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR); + } + + /* + * We have read pgdata path from command line or from configuration file. + * Ensure that pgdata is an absolute path. + */ + if (pgdata != NULL && !is_absolute_path(pgdata)) + elog(ERROR, "-D, --pgdata must be an absolute path"); + + /* Set log path */ + if (log_filename || error_log_filename) + { + if (log_directory) + strcpy(log_path, log_directory); + else + join_path_components(log_path, backup_path, "log"); + } + + /* Sanity check of --backup-id option */ + if (backup_id_string_param != NULL) + { + if (backup_subcmd != RESTORE + && backup_subcmd != VALIDATE + && backup_subcmd != DELETE + && backup_subcmd != SHOW) + elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", + argv[1]); + + current.backup_id = base36dec(backup_id_string_param); + if (current.backup_id == 0) + elog(ERROR, "Invalid backup-id"); + } + + /* Setup stream options. They are used in streamutil.c. */ + if (pgut_dbname != NULL) + dbname = pstrdup(pgut_dbname); + if (host != NULL) + dbhost = pstrdup(host); + if (port != NULL) + dbport = pstrdup(port); + if (username != NULL) + dbuser = pstrdup(username); + + /* setup exclusion list for file search */ + if (!backup_logs) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ + + /* Set 'pg_log' in first empty slot */ + pgdata_exclude_dir[i] = "pg_log"; + } + + if (target_time != NULL && target_xid != NULL) + elog(ERROR, "You can't specify recovery-target-time and recovery-target-xid at the same time"); + + if (num_threads < 1) + num_threads = 1; + + if (backup_subcmd != SET_CONFIG) + { + if (compress_level != DEFAULT_COMPRESS_LEVEL + && compress_alg == NONE_COMPRESS) + elog(ERROR, "Cannot specify compress-level option without compress-alg option"); + } + + if (compress_level < 0 || compress_level > 9) + elog(ERROR, "--compress-level value must be in the range from 0 to 9"); + + /* do actual operation */ + switch (backup_subcmd) + { + case ARCHIVE_PUSH: + return do_archive_push(wal_file_path, wal_file_name); + case ARCHIVE_GET: + return do_archive_get(wal_file_path, wal_file_name); + case ADD_INSTANCE: + return do_add_instance(); + case DELETE_INSTANCE: + return do_delete_instance(); + case INIT: + return do_init(); + case BACKUP: + return do_backup(); + case RESTORE: + return do_restore_or_validate(current.backup_id, + target_time, target_xid, + target_inclusive, target_tli, + true); + case VALIDATE: + if (current.backup_id == 0 && target_time == 0 && target_xid == 0) + return do_validate_all(); + else + return do_restore_or_validate(current.backup_id, + target_time, target_xid, + target_inclusive, target_tli, + false); + case SHOW: + return do_show(current.backup_id); + case DELETE: + if (delete_expired && backup_id_string_param) + elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); + if (delete_expired) + return do_retention_purge(); + else + return do_delete(current.backup_id); + case SHOW_CONFIG: + if (argc > 6) + elog(ERROR, "show-config command doesn't accept any options except -B and --instance"); + return do_configure(true); + case SET_CONFIG: + if (argc == 5) + elog(ERROR, "set-config command requires at least one option"); + return do_configure(false); + } + + return 0; +} + +static void +opt_backup_mode(pgut_option *opt, const char *arg) +{ + current.backup_mode = parse_backup_mode(arg); +} + +static void +opt_log_level(pgut_option *opt, const char *arg) +{ + log_level = parse_log_level(arg); + log_level_defined = true; +} + +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + +void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + compress_alg = parse_compress_alg(arg); +} diff --git a/pg_probackup.h b/src/pg_probackup.h similarity index 83% rename from pg_probackup.h rename to src/pg_probackup.h index 7bf3b559..6e2b42ec 100644 --- a/pg_probackup.h +++ b/src/pg_probackup.h @@ -13,24 +13,34 @@ #include "postgres_fe.h" #include -#include "libpq-fe.h" - -#include "pgut/pgut.h" -#include "access/xlogdefs.h" -#include "access/xlog_internal.h" -#include "catalog/pg_control.h" -#include "utils/pg_crc.h" -#include "parray.h" -#include "datapagemap.h" -#include "storage/bufpage.h" -#include "storage/block.h" -#include "storage/checksum.h" -#include "access/timeline.h" +#include #ifndef WIN32 #include #endif +#include "access/timeline.h" +#include "access/xlogdefs.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum.h" +#include "utils/pg_crc.h" + +#include "utils/parray.h" +#include "utils/pgut.h" + +#include "datapagemap.h" + +# define PG_STOP_BACKUP_TIMEOUT 300 +/* + * Macro needed to parse ptrack. + * NOTE Keep those values syncronised with definitions in ptrack.h + */ +#define PTRACK_BITS_PER_HEAPBLOCK 1 +#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) + /* Directory/File names */ #define DATABASE_DIR "database" #define BACKUPS_DIR "backups" @@ -53,6 +63,14 @@ #define XID_FMT "%u" #endif +typedef enum CompressAlg +{ + NOT_DEFINED_COMPRESS = 0, + NONE_COMPRESS, + PGLZ_COMPRESS, + ZLIB_COMPRESS, +} CompressAlg; + /* Information about single file (or dir) in backup */ typedef struct pgFile { @@ -69,11 +87,13 @@ typedef struct pgFile char *path; /* path of the file */ char *ptrack_path; /* path of the ptrack fork of the relation */ int segno; /* Segment number for ptrack */ - uint64 generation; /* Generation of the compressed file. Set to '-1' - * for non-compressed files. If generation has changed, - * we cannot backup compressed file partially. */ - int is_partial_copy; /* for compressed files. Set to '1' if backed up - * via copy_file_partly() */ + bool is_cfs; /* Flag to distinguish files compressed by CFS*/ + uint64 generation; /* Generation of the compressed file.If generation + * has changed, we cannot backup compressed file + * partially. Has no sense if (is_cfs == false). */ + bool is_partial_copy; /* If the file was backed up via copy_file_partly(). + * Only applies to is_cfs files. */ + CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile uint32 lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ } pgFile; @@ -102,6 +122,10 @@ typedef enum BackupMode typedef enum ProbackupSubcmd { INIT = 0, + ARCHIVE_PUSH, + ARCHIVE_GET, + ADD_INSTANCE, + DELETE_INSTANCE, BACKUP, RESTORE, VALIDATE, @@ -111,6 +135,7 @@ typedef enum ProbackupSubcmd SHOW_CONFIG } ProbackupSubcmd; + /* special values of pgBackup fields */ #define INVALID_BACKUP_ID 0 #define BYTES_INVALID (-1) @@ -124,8 +149,24 @@ typedef struct pgBackupConfig const char *pgport; const char *pguser; + const char *master_host; + const char *master_port; + const char *master_db; + const char *master_user; + int replica_timeout; + + int log_level; + char *log_filename; + char *error_log_filename; + char *log_directory; + int log_rotation_size; + int log_rotation_age; + uint32 retention_redundancy; uint32 retention_window; + + CompressAlg compress_alg; + int compress_level; } pgBackupConfig; /* Information about single backup stored in backup.conf */ @@ -155,6 +196,8 @@ typedef struct pgBackup * BYTES_INVALID means nothing was backed up. */ int64 data_bytes; + /* Size of WAL files in archive needed to restore this backup */ + int64 wal_bytes; /* Fields needed for compatibility check */ uint32 block_size; @@ -217,33 +260,53 @@ extern int cfs_munmap(FileMap* map); #define XLogDataFromLSN(data, xlogid, xrecoff) \ sscanf(data, "%X/%X", xlogid, xrecoff) -/* in probackup.c */ - -/* path configuration */ +/* directory options */ extern char *backup_path; +extern char backup_instance_path[MAXPGPATH]; extern char *pgdata; extern char arclog_path[MAXPGPATH]; -/* current settings */ -extern pgBackup current; -extern ProbackupSubcmd backup_subcmd; - -extern bool smooth_checkpoint; +/* common options */ extern int num_threads; extern bool stream_wal; -extern bool from_replica; extern bool progress; + +/* backup options */ +extern bool smooth_checkpoint; +extern uint32 archive_timeout; +extern bool from_replica; +extern const char *master_db; +extern const char *master_host; +extern const char *master_port; +extern const char *master_user; +extern uint32 replica_timeout; + +/* delete options */ extern bool delete_wal; extern bool delete_expired; extern bool apply_to_all; extern bool force_delete; -extern uint32 archive_timeout; - -extern uint64 system_identifier; +/* retention options */ extern uint32 retention_redundancy; extern uint32 retention_window; +/* compression options */ +extern CompressAlg compress_alg; +extern int compress_level; + +#define DEFAULT_COMPRESS_LEVEL 6 + +extern CompressAlg parse_compress_alg(const char *arg); +extern const char* deparse_compress_alg(int alg); +/* other options */ +extern char *instance_name; +extern uint64 system_identifier; + +/* current settings */ +extern pgBackup current; +extern ProbackupSubcmd backup_subcmd; + /* in dir.c */ /* exclude directory list for $PGDATA file listing */ extern const char *pgdata_exclude_dir[]; @@ -275,6 +338,12 @@ extern void opt_tablespace_map(pgut_option *opt, const char *arg); /* in init.c */ extern int do_init(void); +extern int do_add_instance(void); + +/* in archive.c */ +extern int do_archive_push(char *wal_file_path, char *wal_file_name); +extern int do_archive_get(char *wal_file_path, char *wal_file_name); + /* in configure.c */ extern int do_configure(bool show_only); @@ -289,6 +358,7 @@ extern int do_show(time_t requested_backup_id); /* in delete.c */ extern int do_delete(time_t backup_id); extern int do_retention_purge(void); +extern int do_delete_instance(void); /* in fetch.c */ extern char *slurpFile(const char *datadir, @@ -302,6 +372,7 @@ extern void help_command(char *command); /* in validate.c */ extern void pgBackupValidate(pgBackup* backup); +extern int do_validate_all(void); /* in catalog.c */ extern pgBackup *read_backup(time_t timestamp); @@ -314,6 +385,8 @@ extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void pgBackupWriteBackupControlFile(pgBackup *backup); extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); +extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); @@ -350,12 +423,12 @@ extern void restore_data_file(const char *from_root, const char *to_root, pgFile *file, pgBackup *backup); extern void restore_compressed_file(const char *from_root, const char *to_root, pgFile *file); -extern bool is_compressed_data_file(pgFile *file); extern bool backup_compressed_file_partially(pgFile *file, void *arg, size_t *skip_size); extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern void copy_wal_file(const char *from_root, const char *to_root); extern bool copy_file_partly(const char *from_root, const char *to_root, pgFile *file, size_t skip_size); @@ -389,7 +462,7 @@ extern XLogRecPtr get_last_ptrack_lsn(void); extern uint32 get_data_checksum_version(bool safe); extern char *base36enc(long unsigned int value); extern long unsigned int base36dec(const char *text); -extern uint64 get_system_identifier(void); +extern uint64 get_system_identifier(char *pgdata); extern pg_time_t timestamptz_to_time_t(TimestampTz t); extern void pgBackup_init(pgBackup *backup); diff --git a/restore.c b/src/restore.c similarity index 92% rename from restore.c rename to src/restore.c index 617cdb4f..cc012a9d 100644 --- a/restore.c +++ b/src/restore.c @@ -90,8 +90,8 @@ do_restore_or_validate(time_t target_backup_id, pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; pgBackup *base_full_backup = NULL; - int dest_backup_index; - int base_full_backup_index; + int dest_backup_index = 0; + int base_full_backup_index = 0; char *action = is_restore ? "Restore":"Validate"; if (is_restore) @@ -122,7 +122,7 @@ do_restore_or_validate(time_t target_backup_id, timelines = readTimeLineHistory_probackup(target_tli); } - /* Find backup range we should restore. */ + /* Find backup range we should restore or validate. */ for (i = 0; i < parray_num(backups); i++) { current_backup = (pgBackup *) parray_get(backups, i); @@ -141,7 +141,7 @@ do_restore_or_validate(time_t target_backup_id, { if (current_backup->status != BACKUP_STATUS_OK) elog(ERROR, "Backup %s has status: %s", - base36enc(current_backup->status), status2str(current_backup->status)); + base36enc(current_backup->start_time), status2str(current_backup->status)); if (target_tli) { @@ -217,40 +217,43 @@ do_restore_or_validate(time_t target_backup_id, pgBackupValidate(backup); } + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->recovery_target_time, + rt->recovery_target_xid, base_full_backup->tli); + + /* We ensured that all backups are valid, now restore if required */ if (is_restore) { + pgBackup *backup; for (i = base_full_backup_index; i >= dest_backup_index; i--) { - pgBackup *backup = (pgBackup *) parray_get(backups, i); + backup = (pgBackup *) parray_get(backups, i); if (backup->status == BACKUP_STATUS_OK) restore_backup(backup); else elog(ERROR, "backup %s is not valid", base36enc(backup->start_time)); } - } - /* - * Delete files which are not in dest backup file list. Files which were - * deleted between previous and current backup are not in the list. - */ - if (is_restore) - { - pgBackup *dest_backup = (pgBackup *) parray_get(backups, dest_backup_index); + /* + * Delete files which are not in dest backup file list. Files which were + * deleted between previous and current backup are not in the list. + */ if (dest_backup->backup_mode != BACKUP_MODE_FULL) remove_deleted_files(dest_backup); - } - if (!dest_backup->stream - || (target_time != NULL || target_xid != NULL)) - { - if (is_restore) + /* Create recovery.conf with given recovery target parameters */ + if (!dest_backup->stream + || (target_time != NULL || target_xid != NULL)) + { create_recovery_conf(target_backup_id, target_time, target_xid, target_inclusive, target_tli); - else - validate_wal(dest_backup, arclog_path, rt->recovery_target_time, - rt->recovery_target_xid, base_full_backup->tli); + } } /* cleanup */ @@ -269,7 +272,7 @@ void restore_backup(pgBackup *backup) { char timestamp[100]; - char backup_path[MAXPGPATH]; + char this_backup_path[MAXPGPATH]; char database_path[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; @@ -296,9 +299,10 @@ restore_backup(pgBackup *backup) /* * Restore backup directories. + * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id */ - pgBackupGetPath(backup, backup_path, lengthof(backup_path), NULL); - restore_directories(pgdata, backup_path); + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + restore_directories(pgdata, this_backup_path); /* * Get list of files which need to be restored. @@ -332,8 +336,7 @@ restore_backup(pgBackup *backup) arg->files = files; arg->backup = backup; - if (verbose) - elog(LOG, "Start thread for num:%li", parray_num(files)); + elog(LOG, "Start thread for num:%li", parray_num(files)); restore_threads_args[i] = arg; pthread_create(&restore_threads[i], NULL, (void *(*)(void *)) restore_files, arg); @@ -350,7 +353,7 @@ restore_backup(pgBackup *backup) parray_walk(files, pgFileFree); parray_free(files); - if (verbose) + if (log_level <= LOG) { char *backup_id; @@ -371,11 +374,9 @@ remove_deleted_files(pgBackup *backup) { parray *files; parray *files_restored; - char database_path[MAXPGPATH]; char filelist_path[MAXPGPATH]; int i; - pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); /* Read backup's filelist using target database path as base path */ files = dir_read_file_list(pgdata, filelist_path); @@ -395,7 +396,7 @@ remove_deleted_files(pgBackup *backup) if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { pgFileDelete(file); - if (verbose) + if (log_level <= LOG) elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); } } @@ -449,7 +450,7 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) /* Extract link name from relative path */ link_sep = first_dir_separator(link_ptr); - if (link_sep) + if (link_sep != NULL) { int len = link_sep - link_ptr; strncpy(link_name, link_ptr, len); @@ -483,8 +484,12 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) */ if (strcmp(dir_created, linked_path) == 0) { - /* Create rest of directories */ - if (link_sep && (link_sep + 1)) + /* + * Create rest of directories. + * First check is there any directory name after + * separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') goto create_directory; else continue; @@ -527,8 +532,11 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) /* Save linked directory */ set_tablespace_created(link_name, linked_path); - /* Create rest of directories */ - if (link_sep && (link_sep + 1)) + /* + * Create rest of directories. + * First check is there any directory name after separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') goto create_directory; continue; @@ -560,7 +568,7 @@ create_directory: static void check_tablespace_mapping(pgBackup *backup) { - char backup_path[MAXPGPATH]; + char this_backup_path[MAXPGPATH]; parray *links; size_t i; TablespaceListCell *cell; @@ -568,10 +576,17 @@ check_tablespace_mapping(pgBackup *backup) links = parray_new(); - pgBackupGetPath(backup, backup_path, lengthof(backup_path), NULL); - read_tablespace_map(links, backup_path); + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, this_backup_path); - elog(LOG, "check tablespace directories of backup %s", base36enc(backup->start_time)); + if (log_level <= LOG) + { + char *backup_id; + + backup_id = base36enc(backup->start_time); + elog(LOG, "check tablespace directories of backup %s", backup_id); + pfree(backup_id); + } /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ for (cell = tablespace_dirs.head; cell; cell = cell->next) @@ -665,10 +680,15 @@ restore_files(void *arg) continue; } - /* restore file */ + /* + * restore the file. + * We treat datafiles separately, cause they were backed up block by + * block and have BackupPageHeader meta information, so we cannot just + * copy the file from backup. + */ if (file->is_datafile) { - if (is_compressed_data_file(file)) + if (file->is_cfs) restore_compressed_file(from_root, pgdata, file); else restore_data_file(from_root, pgdata, file, arguments->backup); @@ -703,7 +723,8 @@ create_recovery_conf(time_t backup_id, fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", PROGRAM_VERSION); - fprintf(fp, "restore_command = 'cp %s/%%f %%p'\n", arclog_path); + fprintf(fp, "restore_command = 'pg_probackup archive-get -B %s --instance %s --wal-file-path %%p --wal-file-name %%f'\n", + backup_path, instance_name); if (target_time) fprintf(fp, "recovery_target_time = '%s'\n", target_time); @@ -866,7 +887,6 @@ satisfy_timeline(const parray *timelines, const pgBackup *backup) /* * Get recovery options in the string format, parse them * and fill up the pgRecoveryTarget structure. - * TODO move arguments parsing and validation to getopt. */ pgRecoveryTarget * parseRecoveryTargetOptions(const char *target_time, diff --git a/show.c b/src/show.c similarity index 71% rename from show.c rename to src/show.c index 98006602..92c6da3c 100644 --- a/show.c +++ b/src/show.c @@ -10,19 +10,74 @@ #include "pg_probackup.h" #include +#include +#include +#include + static void show_backup_list(FILE *out, parray *backup_list); static void show_backup_detail(FILE *out, pgBackup *backup); - -/* - * If 'requested_backup_id' is INVALID_BACKUP_ID, show brief meta information - * about all backups in the backup catalog. - * If valid backup id is passed, show detailed meta information - * about specified backup. - */ +static int do_show_instance(time_t requested_backup_id); int do_show(time_t requested_backup_id) +{ + + if (instance_name == NULL + && requested_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "You must specify --instance to use --backup_id option"); + + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + fprintf(stdout, "\nBACKUP INSTANCE '%s'\n", instance_name); + do_show_instance(0); + } + return 0; + } + else + return do_show_instance(requested_backup_id); +} + +/* + * If 'requested_backup_id' is INVALID_BACKUP_ID, show brief meta information + * about all backups in the backup instance. + * If valid backup id is passed, show detailed meta information + * about specified backup. + */ +static int +do_show_instance(time_t requested_backup_id) { if (requested_backup_id != INVALID_BACKUP_ID) { @@ -107,7 +162,6 @@ pretty_size(int64 size, char *buf, size_t len) } } -/* TODO Add comment */ static TimeLineID get_parent_tli(TimeLineID child_tli) { @@ -170,10 +224,11 @@ show_backup_list(FILE *out, parray *backup_list) { int i; + /* if you add new fields here, fix the header */ /* show header */ - fputs("====================================================================================================================\n", out); - fputs("ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n", out); - fputs("====================================================================================================================\n", out); + fputs("===============================================================================================================================\n", out); + fputs(" Instance ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n", out); + fputs("===============================================================================================================================\n", out); for (i = 0; i < parray_num(backup_list); i++) { @@ -202,8 +257,8 @@ show_backup_list(FILE *out, parray *backup_list) parent_tli = get_parent_tli(backup->tli); backup_id = base36enc(backup->start_time); - fprintf(out, "%-6s %-19s %-6s %-7s %3d / %-3d %5s %6s %2X/%08X %2X/%08X %-8s\n", - backup_id, + fprintf(out, " %-11s %-6s %-19s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, backup_id, timestamp, pgBackupGetBackupMode(backup), backup->stream ? "STREAM": "ARCHIVE", diff --git a/status.c b/src/status.c similarity index 100% rename from status.c rename to src/status.c diff --git a/util.c b/src/util.c similarity index 95% rename from util.c rename to src/util.c index 0901f7c7..bf9b6db8 100644 --- a/util.c +++ b/src/util.c @@ -27,7 +27,7 @@ base36enc(long unsigned int value) buffer[--offset] = base36[value % 36]; } while (value /= 36); - return strdup(&buffer[offset]); // warning: this must be free-d by the user + return strdup(&buffer[offset]); /* warning: this must be free-d by the user */ } long unsigned int @@ -75,7 +75,9 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) checkControlFile(ControlFile); } -/* TODO Add comment */ +/* + * Get lsn of the moment when ptrack was enabled the last time. + */ XLogRecPtr get_last_ptrack_lsn(void) { @@ -114,14 +116,14 @@ get_current_timeline(bool safe) } uint64 -get_system_identifier(void) +get_system_identifier(char *pgdata_path) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata, "global/pg_control", &size, false); + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); diff --git a/src/utils/logger.c b/src/utils/logger.c new file mode 100644 index 00000000..ddb90d55 --- /dev/null +++ b/src/utils/logger.c @@ -0,0 +1,532 @@ +/*------------------------------------------------------------------------- + * + * logger.c: - log events into log file or stderr. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "pgut.h" + +/* Logger parameters */ + +int log_level = INFO; +bool log_level_defined = false; + +char *log_filename = NULL; +char *error_log_filename = NULL; +char *log_directory = NULL; +/* + * If log_path is empty logging is not initialized. + * We will log only into stderr + */ +char log_path[MAXPGPATH] = ""; + +/* Maximum size of an individual log file in kilobytes */ +int log_rotation_size = 0; +/* Maximum lifetime of an individual log file in minutes */ +int log_rotation_age = 0; + +/* Implementation for logging.h */ + +typedef enum +{ + PG_DEBUG, + PG_PROGRESS, + PG_WARNING, + PG_FATAL +} eLogType; + +void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); + +static void elog_internal(int elevel, const char *fmt, va_list args) + pg_attribute_printf(2, 0); + +/* Functions to work with log files */ +static void open_logfile(FILE **file, const char *filename_format); +static void release_logfile(void); +static char *logfile_getname(const char *format, time_t timestamp); +static FILE *logfile_open(const char *filename, const char *mode); + +/* Static variables */ + +static FILE *log_file = NULL; +static FILE *error_log_file = NULL; + +static bool exit_hook_registered = false; +/* Logging to file is in progress */ +static bool logging_to_file = false; + +static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void +write_elevel(FILE *stream, int elevel) +{ + switch (elevel) + { + case LOG: + fputs("LOG: ", stream); + break; + case INFO: + fputs("INFO: ", stream); + break; + case NOTICE: + fputs("NOTICE: ", stream); + break; + case WARNING: + fputs("WARNING: ", stream); + break; + case ERROR: + fputs("ERROR: ", stream); + break; + case FATAL: + fputs("FATAL: ", stream); + break; + case PANIC: + fputs("PANIC: ", stream); + break; + default: + elog(ERROR, "invalid logging level: %d", elevel); + break; + } +} + +/* + * Logs to stderr or to log file and exit if ERROR or FATAL. + * + * Actual implementation for elog() and pg_log(). + */ +static void +elog_internal(int elevel, const char *fmt, va_list args) +{ + bool write_to_file, + write_to_error_log, + write_to_stderr; + va_list error_args, + std_args; + + write_to_file = log_path[0] != '\0' && !logging_to_file && + (log_filename || error_log_filename); + + /* + * There is no need to lock if this is elog() from upper elog() and + * logging is not initialized. + */ + if (write_to_file) + pthread_mutex_lock(&log_file_mutex); + + write_to_error_log = + elevel >= ERROR && error_log_filename && write_to_file; + write_to_stderr = elevel >= ERROR || !write_to_file; + + /* We need copy args only if we need write to error log file */ + if (write_to_error_log) + va_copy(error_args, args); + /* + * We need copy args only if we need write to stderr. But do not copy args + * if we need to log only to stderr. + */ + if (write_to_stderr && write_to_file) + va_copy(std_args, args); + + /* + * Write message to log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (log_filename && write_to_file) + { + logging_to_file = true; + + if (log_file == NULL) + open_logfile(&log_file, log_filename); + + write_elevel(log_file, elevel); + + vfprintf(log_file, fmt, args); + fputc('\n', log_file); + fflush(log_file); + + logging_to_file = false; + } + + /* + * Write error message to error log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_error_log) + { + logging_to_file = true; + + if (error_log_file == NULL) + open_logfile(&error_log_file, error_log_filename); + + write_elevel(error_log_file, elevel); + + vfprintf(error_log_file, fmt, error_args); + fputc('\n', error_log_file); + fflush(error_log_file); + + logging_to_file = false; + va_end(error_args); + } + + /* + * Write to stderr if the message was not written to log file. + * Write to stderr if the message level is greater than WARNING anyway. + */ + if (write_to_stderr) + { + write_elevel(stderr, elevel); + + if (write_to_file) + vfprintf(stderr, fmt, std_args); + else + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + if (write_to_file) + va_end(std_args); + } + + if (write_to_file) + pthread_mutex_unlock(&log_file_mutex); + + /* Exit with code if it is an error */ + if (elevel > WARNING) + exit(elevel); +} + +/* + * Logs to stderr or to log file and exit if ERROR or FATAL. + */ +void +elog(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, fmt, args); + va_end(args); +} + +/* + * Implementation of pg_log() from logging.h. + */ +void +pg_log(eLogType type, const char *fmt, ...) +{ + va_list args; + int elevel = INFO; + + /* Transform logging level from eLogType to utils/logger.h levels */ + switch (type) + { + case PG_DEBUG: + elevel = LOG; + break; + case PG_PROGRESS: + elevel = INFO; + break; + case PG_WARNING: + elevel = WARNING; + break; + case PG_FATAL: + elevel = ERROR; + break; + default: + elog(ERROR, "invalid logging level: %d", type); + break; + } + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, fmt, args); + va_end(args); +} + +/* + * Parses string representation of log level. + */ +int +parse_log_level(const char *level) +{ + const char *v = level; + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*v)) + v++; + len = strlen(v); + + if (len == 0) + elog(ERROR, "log-level is empty"); + + if (pg_strncasecmp("verbose", v, len) == 0) + return VERBOSE; + else if (pg_strncasecmp("log", v, len) == 0) + return LOG; + else if (pg_strncasecmp("info", v, len) == 0) + return INFO; + else if (pg_strncasecmp("notice", v, len) == 0) + return NOTICE; + else if (pg_strncasecmp("warning", v, len) == 0) + return WARNING; + else if (pg_strncasecmp("error", v, len) == 0) + return ERROR; + else if (pg_strncasecmp("fatal", v, len) == 0) + return FATAL; + else if (pg_strncasecmp("panic", v, len) == 0) + return PANIC; + + /* Log level is invalid */ + elog(ERROR, "invalid log-level \"%s\"", level); + return 0; +} + +/* + * Converts integer representation of log level to string. + */ +const char * +deparse_log_level(int level) +{ + switch (level) + { + case VERBOSE: + return "VERBOSE"; + case LOG: + return "LOG"; + case INFO: + return "INFO"; + case NOTICE: + return "NOTICE"; + case WARNING: + return "WARNING"; + case ERROR: + return "ERROR"; + case FATAL: + return "FATAL"; + case PANIC: + return "PANIC"; + default: + elog(ERROR, "invalid log-level %d", level); + } + + return NULL; +} + +/* + * Construct logfile name using timestamp information. + * + * Result is palloc'd. + */ +static char * +logfile_getname(const char *format, time_t timestamp) +{ + char *filename; + size_t len; + struct tm *tm = localtime(×tamp); + + if (log_path[0] == '\0') + elog(ERROR, "logging path is not set"); + + filename = (char *) palloc(MAXPGPATH); + + snprintf(filename, MAXPGPATH, "%s/", log_path); + + len = strlen(filename); + + /* Treat log_filename as a strftime pattern */ + if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) + elog(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); + + return filename; +} + +/* + * Open a new log file. + */ +static FILE * +logfile_open(const char *filename, const char *mode) +{ + FILE *fh; + + /* + * Create log directory if not present; ignore errors + */ + mkdir(log_path, S_IRWXU); + + fh = fopen(filename, mode); + + if (fh) + setvbuf(fh, NULL, PG_IOLBF, 0); + else + { + int save_errno = errno; + + elog(FATAL, "could not open log file \"%s\": %s", + filename, strerror(errno)); + errno = save_errno; + } + + return fh; +} + +/* + * Open the log file. + */ +static void +open_logfile(FILE **file, const char *filename_format) +{ + char *filename; + char control[MAXPGPATH]; + struct stat st; + FILE *control_file; + time_t cur_time = time(NULL); + bool rotation_requested = false, + logfile_exists = false; + + filename = logfile_getname(filename_format, cur_time); + + /* "log_path" was checked in logfile_getname() */ + snprintf(control, MAXPGPATH, "%s.rotation", filename); + + if (stat(filename, &st) == -1) + { + if (errno == ENOENT) + { + /* There is no file "filename" and rotation does not need */ + goto logfile_open; + } + else + elog(ERROR, "cannot stat log file \"%s\": %s", + filename, strerror(errno)); + } + /* Found log file "filename" */ + logfile_exists = true; + + /* First check for rotation */ + if (log_rotation_size > 0 || log_rotation_age > 0) + { + /* Check for rotation by age */ + if (log_rotation_age > 0) + { + struct stat control_st; + + if (stat(control, &control_st) == -1) + { + if (errno != ENOENT) + elog(ERROR, "cannot stat rotation file \"%s\": %s", + control, strerror(errno)); + } + else + { + char buf[1024]; + + control_file = fopen(control, "r"); + if (control_file == NULL) + elog(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + if (fgets(buf, lengthof(buf), control_file)) + { + time_t creation_time; + + if (!parse_int64(buf, (int64 *) &creation_time)) + elog(ERROR, "rotation file \"%s\" has wrong " + "creation timestamp \"%s\"", + control, buf); + /* Parsed creation time */ + + rotation_requested = (cur_time - creation_time) > + /* convert to seconds */ + log_rotation_age * 60; + } + else + elog(ERROR, "cannot read creation timestamp from " + "rotation file \"%s\"", control); + + fclose(control_file); + } + } + + /* Check for rotation by size */ + if (!rotation_requested && log_rotation_size > 0) + rotation_requested = st.st_size >= + /* convert to bytes */ + log_rotation_size * 1024L; + } + +logfile_open: + if (rotation_requested) + *file = logfile_open(filename, "w"); + else + *file = logfile_open(filename, "a"); + pfree(filename); + + /* Rewrite rotation control file */ + if (rotation_requested || !logfile_exists) + { + time_t timestamp = time(NULL); + + control_file = fopen(control, "w"); + if (control_file == NULL) + elog(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + fprintf(control_file, "%ld", timestamp); + + fclose(control_file); + } + + /* + * Arrange to close opened file at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(release_logfile); + exit_hook_registered = true; + } +} + +/* + * Closes opened file. + */ +static void +release_logfile(void) +{ + if (log_file) + { + fclose(log_file); + log_file = NULL; + } + if (error_log_file) + { + fclose(error_log_file); + error_log_file = NULL; + } +} diff --git a/src/utils/logger.h b/src/utils/logger.h new file mode 100644 index 00000000..5961dfb0 --- /dev/null +++ b/src/utils/logger.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * logger.h: - prototypes of logger functions. + * + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef LOGGER_H +#define LOGGER_H + +#include "postgres_fe.h" + +/* Log level */ +#define VERBOSE (-5) +#define LOG (-4) +#define INFO (-3) +#define NOTICE (-2) +#define WARNING (-1) +#define ERROR 1 +#define FATAL 2 +#define PANIC 3 + +/* Logger parameters */ + +extern int log_level; +extern bool log_level_defined; + +extern char *log_filename; +extern char *error_log_filename; +extern char *log_directory; +extern char log_path[MAXPGPATH]; + +extern int log_rotation_size; +extern int log_rotation_age; + +#undef elog +extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); + +extern int parse_log_level(const char *level); +extern const char *deparse_log_level(int level); + +#endif /* LOGGER_H */ diff --git a/parray.c b/src/utils/parray.c similarity index 99% rename from parray.c rename to src/utils/parray.c index f2131093..a9ba7c8e 100644 --- a/parray.c +++ b/src/utils/parray.c @@ -7,7 +7,7 @@ *------------------------------------------------------------------------- */ -#include "pg_probackup.h" +#include "src/pg_probackup.h" /* members of struct parray are hidden from client. */ struct parray diff --git a/parray.h b/src/utils/parray.h similarity index 100% rename from parray.h rename to src/utils/parray.h diff --git a/pgut/pgut.c b/src/utils/pgut.c similarity index 81% rename from pgut/pgut.c rename to src/utils/pgut.c index a129e877..03206725 100644 --- a/pgut/pgut.c +++ b/src/utils/pgut.c @@ -2,7 +2,8 @@ * * pgut.c * - * Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional * *------------------------------------------------------------------------- */ @@ -15,6 +16,7 @@ #include #include +#include "logger.h" #include "pgut.h" /* old gcc doesn't have LLONG_MAX. */ @@ -33,8 +35,6 @@ const char *host = NULL; const char *port = NULL; const char *username = NULL; char *password = NULL; -bool verbose = false; -bool quiet = false; bool prompt_password = true; /* Database connections */ @@ -46,16 +46,6 @@ static bool in_cleanup = false; static bool parse_pair(const char buffer[], char key[], char value[]); -typedef enum -{ - PG_DEBUG, - PG_PROGRESS, - PG_WARNING, - PG_FATAL -} eLogType; - -void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); - /* Connection routines */ static void init_cancel_handler(void); static void on_before_exec(PGconn *conn); @@ -65,6 +55,75 @@ static void on_cleanup(void); static void exit_or_abort(int exitcode); static const char *get_username(void); +/* + * Unit conversion tables. + * + * Copied from guc.c. + */ +#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ + +typedef struct +{ + char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or + * "min" */ + int base_unit; /* OPTION_UNIT_XXX */ + int multiplier; /* If positive, multiply the value with this + * for unit -> base_unit conversion. If + * negative, divide (with the absolute value) */ +} unit_conversion; + +static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; + +static const unit_conversion memory_unit_conversion_table[] = +{ + {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, + {"GB", OPTION_UNIT_KB, 1024 * 1024}, + {"MB", OPTION_UNIT_KB, 1024}, + {"kB", OPTION_UNIT_KB, 1}, + + {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XSEGS, (1024 * 1024 * 1024) / (XLOG_SEG_SIZE / 1024)}, + {"GB", OPTION_UNIT_XSEGS, (1024 * 1024) / (XLOG_SEG_SIZE / 1024)}, + {"MB", OPTION_UNIT_XSEGS, -(XLOG_SEG_SIZE / (1024 * 1024))}, + {"kB", OPTION_UNIT_XSEGS, -(XLOG_SEG_SIZE / 1024)}, + + {""} /* end of table marker */ +}; + +static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; + +static const unit_conversion time_unit_conversion_table[] = +{ + {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, + {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, + {"min", OPTION_UNIT_MS, 1000 * 60}, + {"s", OPTION_UNIT_MS, 1000}, + {"ms", OPTION_UNIT_MS, 1}, + + {"d", OPTION_UNIT_S, 60 * 60 * 24}, + {"h", OPTION_UNIT_S, 60 * 60}, + {"min", OPTION_UNIT_S, 60}, + {"s", OPTION_UNIT_S, 1}, + {"ms", OPTION_UNIT_S, -1000}, + + {"d", OPTION_UNIT_MIN, 60 * 24}, + {"h", OPTION_UNIT_MIN, 60}, + {"min", OPTION_UNIT_MIN, 1}, + {"s", OPTION_UNIT_MIN, -60}, + {"ms", OPTION_UNIT_MIN, -1000 * 60}, + + {""} /* end of table marker */ +}; + static size_t option_length(const pgut_option opts[]) { @@ -115,7 +174,7 @@ option_find(int c, pgut_option opts1[]) static void assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) { - const char *message; + const char *message; if (opt == NULL) { @@ -135,6 +194,8 @@ assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) } else { + pgut_optsrc orig_source = opt->source; + /* can be overwritten if non-command line source */ opt->source = src; @@ -177,10 +238,13 @@ assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) message = "a 64bit unsigned integer"; break; case 's': - if (opt->source != SOURCE_DEFAULT) + if (orig_source != SOURCE_DEFAULT) free(*(char **) opt->var); *(char **) opt->var = pgut_strdup(optarg); - return; + if (strcmp(optarg,"") != 0) + return; + message = "a valid string. But provided: "; + break; case 't': if (parse_time(optarg, opt->var)) return; @@ -481,6 +545,129 @@ parse_time(const char *value, time_t *time) return true; } +/* + * Convert a value from one of the human-friendly units ("kB", "min" etc.) + * to the given base unit. 'value' and 'unit' are the input value and unit + * to convert from. The converted value is stored in *base_value. + * + * Returns true on success, false if the input unit is not recognized. + */ +static bool +convert_to_base_unit(int64 value, const char *unit, + int base_unit, int64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Try to parse value as an integer. The accepted formats are the + * usual decimal, octal, or hexadecimal formats, optionally followed by + * a unit name if "flags" indicates a unit is allowed. + * + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable + * HINT message, or NULL if no hint provided. + */ +bool +parse_int(const char *value, int *result, int flags, const char **hintmsg) +{ + int64 val; + char *endptr; + + /* To suppress compiler warnings, always set output params */ + if (result) + *result = 0; + if (hintmsg) + *hintmsg = NULL; + + /* We assume here that int64 is at least as wide as long */ + errno = 0; + val = strtol(value, &endptr, 0); + + if (endptr == value) + return false; /* no HINT for integer syntax error */ + + if (errno == ERANGE || val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + /* Handle possible unit */ + if (*endptr != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*endptr != '\0' && !isspace((unsigned char) *endptr) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(endptr++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + if (*endptr == '\0') + converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), + &val); + if (!converted) + { + /* invalid unit, or garbage after the unit; set hint and fail. */ + if (hintmsg) + { + if (flags & OPTION_UNIT_MEMORY) + *hintmsg = memory_units_hint; + else + *hintmsg = time_units_hint; + } + return false; + } + + /* Check for overflow due to units conversion */ + if (val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + } + + if (result) + *result = (int) val; + return true; +} + static char * longopts_to_optstring(const struct option opts[]) { @@ -828,15 +1015,24 @@ prompt_for_password(const char *username) PGconn * pgut_connect(const char *dbname) +{ + return pgut_connect_extended(host, port, dbname, username, password); +} + +PGconn * +pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login, const char *pwd) { PGconn *conn; + if (interrupted && !in_cleanup) elog(ERROR, "interrupted"); /* Start the connection. Loop until we have a password if requested by backend. */ for (;;) { - conn = PQsetdbLogin(host, port, NULL, NULL, dbname, username, password); + conn = PQsetdbLogin(pghost, pgport, NULL, NULL, + dbname, login, pwd); if (PQstatus(conn) == CONNECTION_OK) return conn; @@ -896,7 +1092,7 @@ pgut_execute(PGconn* conn, const char *query, int nParams, const char **params) elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (verbose) + if (log_level <= LOG) { int i; @@ -945,7 +1141,7 @@ pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (verbose) + if (log_level <= LOG) { int i; @@ -978,6 +1174,24 @@ pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int return true; } +void +pgut_cancel(PGconn* conn) +{ + PGcancel *cancel_conn = PQgetCancel(conn); + char errbuf[256]; + + if (cancel_conn != NULL) + { + if (PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + elog(WARNING, "Cancel request sent"); + else + elog(WARNING, "Cancel request failed"); + } + + if (cancel_conn) + PQfreeCancel(cancel_conn); +} + int pgut_wait(int num, PGconn *connections[], struct timeval *timeout) { @@ -1032,94 +1246,6 @@ pgut_wait(int num, PGconn *connections[], struct timeval *timeout) return -1; } -/* - * elog - log to stderr and exit if ERROR or FATAL - */ -void -elog(int elevel, const char *fmt, ...) -{ - va_list args; - - if (!verbose && elevel <= LOG) - return; - if (quiet && elevel < WARNING) - return; - - switch (elevel) - { - case LOG: - fputs("LOG: ", stderr); - break; - case INFO: - fputs("INFO: ", stderr); - break; - case NOTICE: - fputs("NOTICE: ", stderr); - break; - case WARNING: - fputs("WARNING: ", stderr); - break; - case FATAL: - fputs("FATAL: ", stderr); - break; - case PANIC: - fputs("PANIC: ", stderr); - break; - default: - if (elevel >= ERROR) - fputs("ERROR: ", stderr); - break; - } - - va_start(args, fmt); - vfprintf(stderr, fmt, args); - fputc('\n', stderr); - fflush(stderr); - va_end(args); - - if (elevel > 0) - exit_or_abort(elevel); -} - -void pg_log(eLogType type, const char *fmt, ...) -{ - va_list args; - - if (!verbose && type <= PG_PROGRESS) - return; - if (quiet && type < PG_WARNING) - return; - - switch (type) - { - case PG_DEBUG: - fputs("DEBUG: ", stderr); - break; - case PG_PROGRESS: - fputs("PROGRESS: ", stderr); - break; - case PG_WARNING: - fputs("WARNING: ", stderr); - break; - case PG_FATAL: - fputs("FATAL: ", stderr); - break; - default: - if (type >= PG_FATAL) - fputs("ERROR: ", stderr); - break; - } - - va_start(args, fmt); - vfprintf(stderr, fmt, args); - fputc('\n', stderr); - fflush(stderr); - va_end(args); - - if (type > 0) - exit_or_abort(type); -} - #ifdef WIN32 static CRITICAL_SECTION cancelConnLock; #endif diff --git a/pgut/pgut.h b/src/utils/pgut.h similarity index 82% rename from pgut/pgut.h rename to src/utils/pgut.h index c19efe04..5ec40589 100644 --- a/pgut/pgut.h +++ b/src/utils/pgut.h @@ -2,7 +2,8 @@ * * pgut.h * - * Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional * *------------------------------------------------------------------------- */ @@ -16,6 +17,8 @@ #include #include +#include "logger.h" + #if !defined(C_H) && !defined(__cplusplus) #ifndef bool typedef char bool; @@ -65,6 +68,22 @@ typedef struct pgut_option typedef void (*pgut_optfn) (pgut_option *opt, const char *arg); typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); +/* + * bit values in "flags" of an option + */ +#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ +#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ +#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ +#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ +#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ + +#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ +#define OPTION_UNIT_S 0x20000 /* value is in seconds */ +#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ +#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ + +#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) + /* * pgut client variables and functions */ @@ -83,8 +102,6 @@ extern const char *host; extern const char *port; extern const char *username; extern char *password; -extern bool verbose; -extern bool quiet; extern bool prompt_password; extern bool interrupted; @@ -99,9 +116,13 @@ extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); * Database connections */ extern PGconn *pgut_connect(const char *dbname); +extern PGconn *pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login, + const char *pwd); extern void pgut_disconnect(PGconn *conn); extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, const char **params); extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); +extern void pgut_cancel(PGconn* conn); extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); extern const char *pgut_get_host(void); @@ -126,23 +147,6 @@ extern char *strdup_trim(const char *str); */ extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); -/* - * elog - */ -#define VERBOSE (-5) -#define LOG (-4) -#define INFO (-3) -#define NOTICE (-2) -#define WARNING (-1) -#define ERROR 1 -#define FATAL 2 -#define PANIC 3 - -#undef elog -extern void -elog(int elevel, const char *fmt, ...) -__attribute__((format(printf, 2, 3))); - /* * Assert */ @@ -189,6 +193,8 @@ extern bool parse_uint32(const char *value, uint32 *result); extern bool parse_int64(const char *value, int64 *result); extern bool parse_uint64(const char *value, uint64 *result); extern bool parse_time(const char *value, time_t *time); +extern bool parse_int(const char *value, int *result, int flags, + const char **hintmsg); #define IsSpace(c) (isspace((unsigned char)(c))) #define IsAlpha(c) (isalpha((unsigned char)(c))) diff --git a/validate.c b/src/validate.c similarity index 54% rename from validate.c rename to src/validate.c index 0ae86228..0795179a 100644 --- a/validate.c +++ b/src/validate.c @@ -12,8 +12,12 @@ #include #include +#include static void pgBackupValidateFiles(void *arg); +static void do_validate_instance(void); + +static bool corrupted_backup_found = false; typedef struct { @@ -36,8 +40,17 @@ pgBackupValidate(pgBackup *backup) validate_files_args *validate_threads_args[num_threads]; int i; + /* We need free() this later */ backup_id_string = base36enc(backup->start_time); + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) + { + elog(INFO, "Backup %s has status %s. Skip validation.", + backup_id_string, status2str(backup->status)); + return; + } + elog(LOG, "Validate backup %s", backup_id_string); if (backup->backup_mode != BACKUP_MODE_FULL && @@ -123,8 +136,12 @@ pgBackupValidateFiles(void *arg) */ if (file->write_size == BYTES_INVALID) continue; - /* We don't compute checksums for compressed data, so skip them */ - if (file->generation != -1) + + /* + * Currently we don't compute checksums for + * cfs_compressed data files, so skip them. + */ + if (file->is_cfs) continue; /* print progress */ @@ -161,3 +178,127 @@ pgBackupValidateFiles(void *arg) } } } + +/* + * Validate all backups in the backup catalog. + * If --instance option was provided, validate only backups of this instance. + */ +int +do_validate_all(void) +{ + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + do_validate_instance(); + } + } + else + { + do_validate_instance(); + } + + if (corrupted_backup_found) + elog(INFO, "Some backups are not valid"); + else + elog(INFO, "All backups are valid"); + + return 0; +} + +/* + * Validate all backups in the given instance of the backup catalog. + */ +static void +do_validate_instance(void) +{ + int i, j; + parray *backups; + pgBackup *current_backup = NULL; + pgBackup *base_full_backup = NULL; + + elog(INFO, "Validate backups of the instance '%s'", instance_name); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + if (backups == NULL) + elog(ERROR, "Failed to get backup list."); + + /* Valiate each backup along with its xlog files. */ + for (i = 0; i < parray_num(backups); i++) + { + char *backup_id; + + current_backup = (pgBackup *) parray_get(backups, i); + backup_id = base36enc(current_backup->start_time); + + elog(INFO, "Validate backup %s", backup_id); + + free(backup_id); + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + j = i+1; + do + { + base_full_backup = (pgBackup *) parray_get(backups, j); + j++; + } + while (base_full_backup->backup_mode != BACKUP_MODE_FULL + && j < parray_num(backups)); + } + else + base_full_backup = current_backup; + + pgBackupValidate(current_backup); + + /* There is no point in wal validation for corrupted backup */ + if (current_backup->status == BACKUP_STATUS_OK) + { + /* Validate corresponding WAL files */ + validate_wal(current_backup, arclog_path, 0, + 0, base_full_backup->tli); + } + + if (current_backup->status != BACKUP_STATUS_OK) + corrupted_backup_found = true; + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); +} diff --git a/tests/__init__.py b/tests/__init__.py index 70b792cb..e6094ad0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,11 +5,17 @@ from . import init_test, option_test, show_test, \ retention_test, ptrack_clean, ptrack_cluster, \ ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ - ptrack_vacuum_full, ptrack_vacuum_truncate + ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ + pgpro688, false_positive, replica def load_tests(loader, tests, pattern): suite = unittest.TestSuite() + suite.addTests(loader.loadTestsFromModule(replica)) +# suite.addTests(loader.loadTestsFromModule(pgpro560)) +# suite.addTests(loader.loadTestsFromModule(pgpro589)) +# suite.addTests(loader.loadTestsFromModule(pgpro688)) +# suite.addTests(loader.loadTestsFromModule(false_positive)) suite.addTests(loader.loadTestsFromModule(init_test)) suite.addTests(loader.loadTestsFromModule(option_test)) suite.addTests(loader.loadTestsFromModule(show_test)) @@ -29,3 +35,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) return suite + + +# ExpectedFailures are bugs, which should be fixed diff --git a/tests/backup_test.py b/tests/backup_test.py index 62c5ceff..b68be8d2 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -1,7 +1,8 @@ import unittest -from os import path, listdir +import os import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from time import sleep +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from testgres import stop_all @@ -9,159 +10,210 @@ class BackupTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(BackupTest, self).__init__(*args, **kwargs) + self.module_name = 'backup' -# @classmethod -# def tearDownClass(cls): -# stop_all() -# @unittest.skip("123") + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-707 def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) # full backup mode - with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + #with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: + # backup_log.write(self.backup_node(node, options=["--verbose"])) + + backup_id = self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] - show_backup = self.show_pb(node)[0] - full_backup_id = show_backup['ID'] self.assertEqual(show_backup['Status'], six.b("OK")) self.assertEqual(show_backup['Mode'], six.b("FULL")) # postmaster.pid and postmaster.opts shouldn't be copied excluded = True - backups_dir = path.join(self.backup_dir(node), "backups") - for backup in listdir(backups_dir): - db_dir = path.join(backups_dir, backup, "database") - for f in listdir(db_dir): - if path.isfile(path.join(db_dir, f)) and \ - (f == "postmaster.pid" or f == "postmaster.opts"): + db_dir = os.path.join(backup_dir, "backups", 'node', backup_id, "database") + for f in os.listdir(db_dir): + if os.path.isfile(os.path.join(db_dir, f)) \ + and (f == "postmaster.pid" or f == "postmaster.opts"): excluded = False self.assertEqual(excluded, True) # page backup mode - with open(path.join(node.logs_dir, "backup_page.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"])) + page_backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") # print self.show_pb(node) - show_backup = self.show_pb(node)[1] + show_backup = self.show_pb(backup_dir, 'node')[1] self.assertEqual(show_backup['Status'], six.b("OK")) self.assertEqual(show_backup['Mode'], six.b("PAGE")) # Check parent backup self.assertEqual( - full_backup_id, - self.show_pb(node, id=show_backup['ID'])["parent-backup-id"]) + backup_id, + self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"]) # ptrack backup mode - with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - show_backup = self.show_pb(node)[2] + show_backup = self.show_pb(backup_dir, 'node')[2] self.assertEqual(show_backup['Status'], six.b("OK")) self.assertEqual(show_backup['Mode'], six.b("PTRACK")) + # Check parent backup + self.assertEqual( + page_backup_id, + self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"]) + node.stop() -# @unittest.skip("123") + # @unittest.skip("skip") def test_smooth_checkpoint(self): """full backup with smooth checkpoint""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - - with open(path.join(node.logs_dir, "backup.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose", "-C"])) - - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node' ,node, options=["-C"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) node.stop() -# @unittest.skip("123") - def test_page_backup_without_full(self): + #@unittest.skip("skip") + def test_incremental_backup_without_full(self): """page-level backup without validated full backup""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) try: - self.backup_pb(node, backup_type="page", options=["--verbose"]) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - pass - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("ERROR")) + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + sleep(1) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR")) node.stop() -# @unittest.skip("123") + @unittest.expectedFailure + # Need to forcibly validate parent + def test_incremental_backup_corrupt_full(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + sleep(1) + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR")) + node.stop() + + # @unittest.skip("skip") def test_ptrack_threads(self): """ptrack multi thread backup mode""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose", "-j", "4"])) + self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) - - with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "-j", "4"])) - - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) node.stop() -# @unittest.skip("123") + # @unittest.skip("skip") def test_ptrack_threads_stream(self): """ptrack multi thread backup mode and stream""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) -# node.append_conf("pg_hba.conf", "local replication all trust") -# node.append_conf("pg_hba.conf", "host replication all 127.0.0.1/32 trust") + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - backup_log.write(self.backup_pb( - node, - backup_type="full", - options=["--verbose", "-j", "4", "--stream"] - )) - - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) - - with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log: - backup_log.write(self.backup_pb( - node, - backup_type="ptrack", - options=["--verbose", "-j", "4", "--stream"] - )) - - self.assertEqual(self.show_pb(node)[1]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK")) node.stop() diff --git a/tests/class_check.py b/tests/class_check.py deleted file mode 100644 index ad6c44af..00000000 --- a/tests/class_check.py +++ /dev/null @@ -1,24 +0,0 @@ -class Base(object): - def __init__(self): - self.a = 10 - def func(self, arg1, arg2): - print 'Child {0}, a = {1}'.format(arg1, arg2) - - -class ChildA(Base): - def __init__(self): - Base.__init__(self) - b = 5 - c = b + self.a - print 'Child A, a = {0}'.format(c) - - -class ChildB(Base): - def __init__(self): - super(ChildB, self).__init__() - b = 6 - c = b + self.a - self.func('B', c) - -#ChildA() -ChildB() diff --git a/tests/class_check1.py b/tests/class_check1.py deleted file mode 100644 index 06759d7b..00000000 --- a/tests/class_check1.py +++ /dev/null @@ -1,15 +0,0 @@ -class Foo(object): - def __init__(self, *value1, **value2): -# do something with the values - print 'I think something is being called here' -# print value1, value2 - - -class MyFoo(Foo): - def __init__(self, *args, **kwargs): -# do something else, don't care about the args - print args, kwargs - super(MyFoo, self).__init__(*args, **kwargs) - - -foo = MyFoo('Python', 2.7, stack='overflow', ololo='lalala') \ No newline at end of file diff --git a/tests/class_check2.py b/tests/class_check2.py deleted file mode 100644 index bf6d0a9f..00000000 --- a/tests/class_check2.py +++ /dev/null @@ -1,23 +0,0 @@ -class Base(object): - def __init__(self): - self.a = 10 - self.b = 1 -# def func(self, arg1, arg2): -# print 'Child {0}, a = {1}'.format(arg1, arg2) - - -class ChildA(Base): - def __init__(self): - Base.__init__(self) - self.b = self.b + 1 - - -class ChildB(ChildA): - def __init__(self): - ChildA.__init__(self) - print 'b = {0}'.format(self.b) -# c = b + self.a - - -#ChildA() -ChildB() diff --git a/tests/delete_test.py b/tests/delete_test.py index 41ea70da..e9c176f0 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -1,7 +1,7 @@ import unittest -from os import path +import os import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from testgres import stop_all import subprocess @@ -10,88 +10,120 @@ class DeleteTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(DeleteTest, self).__init__(*args, **kwargs) + self.module_name = 'delete' -# @classmethod -# def tearDownClass(cls): -# stop_all() -# @unittest.skip("123") + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure def test_delete_full_backups(self): """delete full backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/delete/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - node.pgbench_init() - # full backup mode - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + # full backup + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) - show_backups = self.show_pb(node) + show_backups = self.show_pb(backup_dir, 'node') id_1 = show_backups[0]['ID'] + id_2 = show_backups[1]['ID'] id_3 = show_backups[2]['ID'] - self.delete_pb(node, show_backups[1]['ID']) - show_backups = self.show_pb(node) + self.delete_pb(backup_dir, 'node', id_2) + show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(show_backups[0]['ID'], id_1) self.assertEqual(show_backups[1]['ID'], id_3) node.stop() -# @unittest.skip("123") - def test_delete_increment(self): + def test_delete_increment_page(self): """delete increment and all after him""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/delete/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) # full backup mode - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) - + self.backup_node(backup_dir, 'node', node) # page backup mode - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"])) - + self.backup_node(backup_dir, 'node', node, backup_type="page") # page backup mode - with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"])) - + self.backup_node(backup_dir, 'node', node, backup_type="page") # full backup mode - self.backup_pb(node) - - show_backups = self.show_pb(node) + self.backup_node(backup_dir, 'node', node) + show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(node, show_backups[1]['ID']) + self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) - show_backups = self.show_pb(node) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['Mode'], six.b("FULL")) + self.assertEqual(show_backups[0]['Status'], six.b("OK")) + self.assertEqual(show_backups[1]['Mode'], six.b("FULL")) + self.assertEqual(show_backups[1]['Status'], six.b("OK")) + + node.stop() + + def test_delete_increment_ptrack(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) + + show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) self.assertEqual(show_backups[0]['Mode'], six.b("FULL")) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 53b46eb4..838dd4b2 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -5,36 +5,58 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup version - pg_probackup init -B backup-path -D pgdata-dir + pg_probackup init -B backup-path - pg_probackup set-config -B backup-dir - [-d dbname] [-h host] [-p port] [-U username] - [--retention-redundancy=retention-redundancy]] + pg_probackup set-config -B backup-dir --instance=instance_name + [--log-level=log-level] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] - - pg_probackup show-config -B backup-dir - - pg_probackup backup -B backup-path -b backup-mode - [-D pgdata-dir] [-C] [--stream [-S slot-name]] [--backup-pg-log] - [-j num-threads] [--archive-timeout=archive-timeout] - [--progress] [-q] [-v] [--delete-expired] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] [-d dbname] [-h host] [-p port] [-U username] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] - pg_probackup restore -B backup-dir - [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v] - [--time=time|--xid=xid [--inclusive=boolean]] - [--timeline=timeline] [-T OLDDIR=NEWDIR] + pg_probackup show-config -B backup-dir --instance=instance_name - pg_probackup validate -B backup-dir - [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v] - [--time=time|--xid=xid [--inclusive=boolean]] - [--timeline=timeline] + pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + [-C] [--stream [-S slot-name]] [--backup-pg-log] + [-j num-threads] [--archive-timeout=archive-timeout] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [--progress] [--delete-expired] + [-d dbname] [-h host] [-p port] [-U username] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup restore -B backup-dir --instance=instance_name + [-D pgdata-dir] [-i backup-id] [--progress] + [--time=time|--xid=xid [--inclusive=boolean]] + [--timeline=timeline] [-T OLDDIR=NEWDIR] + + pg_probackup validate -B backup-dir [--instance=instance_name] + [-i backup-id] [--progress] + [--time=time|--xid=xid [--inclusive=boolean]] + [--timeline=timeline] pg_probackup show -B backup-dir - [-i backup-id] + [--instance=instance_name [-i backup-id]] - pg_probackup delete -B backup-dir - [--wal] [-i backup-id | --expired] + pg_probackup delete -B backup-dir --instance=instance_name + [--wal] [-i backup-id | --expired] + + pg_probackup add-instance -B backup-dir -D pgdata-dir + --instance=instance_name + + pg_probackup del-instance -B backup-dir + --instance=instance_name Read the website for details. Report bugs to . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index adc3ad0d..e88cb7c9 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 1.1.11 +pg_probackup 1.1.17 diff --git a/tests/false_positive.py b/tests/false_positive.py new file mode 100644 index 00000000..71e2899f --- /dev/null +++ b/tests/false_positive.py @@ -0,0 +1,155 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class FalsePositive(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(FalsePositive, self).__init__(*args, **kwargs) + self.module_name = 'false_positive' + + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro561(self): + """ + make node with archiving, make stream backup, restore it to node1, + check that archiving is not successful on node1 + """ + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/master".format(fname), + set_archiving=True, + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + master.start() + + self.assertEqual(self.init_pb(master), six.b("")) + id = self.backup_pb(master, backup_type='full', options=["--stream"]) + + node1 = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/node1".format(fname)) + node1.cleanup() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + + self.backup_pb(master, backup_type='page', options=["--stream"]) + self.restore_pb(backup_dir=self.backup_dir(master), data_dir=node1.data_dir) + node1.append_conf('postgresql.auto.conf', 'port = {0}'.format(node1.port)) + node1.start({"-t": "600"}) + + timeline_master = master.get_control_data()["Latest checkpoint's TimeLineID"] + timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual(timeline_master, timeline_node1, "Timelines on Master and Node1 should be equal. This is unexpected") + + archive_command_master = master.safe_psql("postgres", "show archive_command") + archive_command_node1 = node1.safe_psql("postgres", "show archive_command") + self.assertEqual(archive_command_master, archive_command_node1, "Archive command on Master and Node should be equal. This is unexpected") + + res = node1.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + # self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) + if res == six.b(""): + self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') + + master.stop() + node1.stop() + + def pgpro688(self): + """ + make node with archiving, make backup, + get Recovery Time, validate to Recovery Time + Waiting PGPRO-688 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), + set_archiving=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + id = self.backup_pb(node, backup_type='full') + recovery_time = self.show_pb(node, id=id)['recovery-time'] + + # Uncommenting this section will make this test True Positive + #node.psql("postgres", "select pg_create_restore_point('123')") + #node.psql("postgres", "select txid_current()") + #node.psql("postgres", "select pg_switch_xlog()") + #### + + try: + self.validate_pb(node, options=["--time='{0}'".format(recovery_time)]) + self.assertEqual(1, 0, 'Error is expected because We should not be able safely validate "Recovery Time" without wal record with timestamp') + except ProbackupException, e: + self.assertTrue('WARNING: recovery can be done up to time {0}'.format(recovery_time) in e.message) + + node.stop() + + def pgpro702_688(self): + """ + make node without archiving, make stream backup, + get Recovery Time, validate to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + id = self.backup_pb(node, backup_type='full', options=["--stream"]) + recovery_time = self.show_pb(node, id=id)['recovery-time'] + + self.assertIn(six.b("INFO: backup validation completed successfully on"), + self.validate_pb(node, options=["--time='{0}'".format(recovery_time)])) + + def test_validate_wal_lost_segment(self): + """Loose segment located between backups. ExpectedFailure. This is BUG """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), + set_archiving=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + node.start() + self.assertEqual(self.init_pb(node), six.b("")) + self.backup_pb(node, backup_type='full') + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(self.backup_dir(node), "wal") + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = map(int, wals) + os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))) + + + ##### Hole Smokes, Batman! We just lost a wal segment and know nothing about it + ##### We need archive-push ASAP + self.backup_pb(node, backup_type='full') + self.assertTrue('validation completed successfully' in self.validate_pb(node)) + ######## + node.stop() diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..769446ad --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,2 @@ +__all__ = ['ptrack_helpers', 'expected_errors'] +#from . import * \ No newline at end of file diff --git a/tests/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py similarity index 81% rename from tests/ptrack_helpers.py rename to tests/helpers/ptrack_helpers.py index 40c77c1a..b7a7cf3b 100644 --- a/tests/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -7,6 +7,7 @@ import six from testgres import get_new_node import hashlib import re +import pwd idx_ptrack = { @@ -55,8 +56,6 @@ Splitted Body # You can lookup error message and cmdline in exception object attributes class ProbackupException(Exception): def __init__(self, message, cmd): -# print message -# self.message = repr(message).strip("'") self.message = message self.cmd = cmd #need that to make second raise @@ -136,25 +135,34 @@ class ProbackupTest(object): self.test_env["LC_MESSAGES"] = "C" self.test_env["LC_TIME"] = "C" - self.dir_path = os.path.dirname(os.path.realpath(__file__)) + self.helpers_path = os.path.dirname(os.path.realpath(__file__)) + self.dir_path = os.path.abspath(os.path.join(self.helpers_path, os.pardir)) + self.tmp_path = os.path.abspath(os.path.join(self.dir_path, 'tmp_dirs')) try: - os.makedirs(os.path.join(self.dir_path, "tmp_dirs")) + os.makedirs(os.path.join(self.dir_path, 'tmp_dirs')) except: pass self.probackup_path = os.path.abspath(os.path.join( - self.dir_path, - "../pg_probackup" - )) + self.dir_path, "../pg_probackup")) + self.user = self.get_username() def arcwal_dir(self, node): return "%s/backup/wal" % node.base_dir - def backup_dir(self, node): - return os.path.abspath("%s/backup" % node.base_dir) + def backup_dir(self, node=None, path=None): + if node: + return os.path.abspath("{0}/backup".format(node.base_dir)) + if path: + return - def make_simple_node(self, base_dir=None, set_replication=False, - set_archiving=False, initdb_params=[], pg_options={}): - real_base_dir = os.path.join(self.dir_path, base_dir) + def make_simple_node( + self, + base_dir=None, + set_replication=False, + initdb_params=[], + pg_options={}): + + real_base_dir = os.path.join(self.tmp_path, base_dir) shutil.rmtree(real_base_dir, ignore_errors=True) node = get_new_node('test', base_dir=real_base_dir) @@ -172,9 +180,6 @@ class ProbackupTest(object): # Allow replication in pg_hba.conf if set_replication: node.set_replication_conf() - # Setup archiving for node - if set_archiving: - self.set_archiving_conf(node, self.arcwal_dir(node)) return node def create_tblspace_in_node(self, node, tblspc_name, cfs=False): @@ -299,7 +304,8 @@ class ProbackupTest(object): def run_pb(self, command, async=False): try: - #print ' '.join(map(str,[self.probackup_path] + command)) + self.cmd = [' '.join(map(str,[self.probackup_path] + command))] + print self.cmd if async is True: return subprocess.Popen( [self.probackup_path] + command, @@ -308,80 +314,93 @@ class ProbackupTest(object): env=self.test_env ) else: - output = subprocess.check_output( + self.output = subprocess.check_output( [self.probackup_path] + command, stderr=subprocess.STDOUT, env=self.test_env ) if command[0] == 'backup': - if '-q' in command or '--quiet' in command: - return None - elif '-v' in command or '--verbose' in command: - return output - else: - # return backup ID - for line in output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - return line.split()[2] + # return backup ID + for line in self.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] else: - return output + return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output, e.cmd) + raise ProbackupException(e.output, self.cmd) - def init_pb(self, node): + def init_pb(self, backup_dir): + shutil.rmtree(backup_dir, ignore_errors=True) return self.run_pb([ "init", - "-B", self.backup_dir(node), + "-B", backup_dir + ]) + + def add_instance(self, backup_dir, instance, node): + + return self.run_pb([ + "add-instance", + "--instance={0}".format(instance), + "-B", backup_dir, "-D", node.data_dir ]) - def clean_pb(self, node): - shutil.rmtree(self.backup_dir(node), ignore_errors=True) + def del_instance(self, backup_dir, instance, node): - def backup_pb(self, node=None, data_dir=None, backup_dir=None, backup_type="full", options=[], async=False): - if data_dir is None: - data_dir = node.data_dir - if backup_dir is None: - backup_dir = self.backup_dir(node) + return self.run_pb([ + "del-instance", + "--instance={0}".format(instance), + "-B", backup_dir, + "-D", node.data_dir + ]) + + def clean_pb(self, backup_dir): + shutil.rmtree(backup_dir, ignore_errors=True) + + def backup_node(self, backup_dir, instance, node, backup_type="full", options=[], async=False): cmd_list = [ "backup", "-B", backup_dir, - "-D", data_dir, + "-D", node.data_dir, "-p", "%i" % node.port, - "-d", "postgres" + "-d", "postgres", + "--instance={0}".format(instance) ] if backup_type: cmd_list += ["-b", backup_type] return self.run_pb(cmd_list + options, async) - def restore_pb(self, node=None, backup_dir=None, data_dir=None, id=None, options=[]): + def restore_node(self, backup_dir, instance, node=False, data_dir=None, backup_id=None, options=[]): if data_dir is None: data_dir = node.data_dir - if backup_dir is None: - backup_dir = self.backup_dir(node) cmd_list = [ "restore", "-B", backup_dir, - "-D", data_dir + "-D", data_dir, + "--instance={0}".format(instance) ] - if id: - cmd_list += ["-i", id] + if backup_id: + cmd_list += ["-i", backup_id] return self.run_pb(cmd_list + options) - def show_pb(self, node, id=None, options=[], as_text=False): + def show_pb(self, backup_dir, instance=None, backup_id=None, options=[], as_text=False): + backup_list = [] specific_record = {} cmd_list = [ "show", - "-B", self.backup_dir(node), + "-B", backup_dir, ] - if id: - cmd_list += ["-i", id] + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if backup_id: + cmd_list += ["-i", backup_id] if as_text: # You should print it when calling as_text=true @@ -389,7 +408,7 @@ class ProbackupTest(object): # get show result as list of lines show_splitted = self.run_pb(cmd_list + options).splitlines() - if id is None: + if instance is not None and backup_id is None: # cut header(ID, Mode, etc) from show as single string header = show_splitted[1:2][0] # cut backup records from show as single list with string for every backup record @@ -431,40 +450,46 @@ class ProbackupTest(object): specific_record[name.strip()] = var return specific_record - def validate_pb(self, node, id=None, options=[]): + def validate_pb(self, backup_dir, instance=None, backup_id=None, options=[]): + cmd_list = [ "validate", - "-B", self.backup_dir(node), + "-B", backup_dir ] - if id: - cmd_list += ["-i", id] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] - # print(cmd_list) return self.run_pb(cmd_list + options) - def delete_pb(self, node, id=None, options=[]): + def delete_pb(self, backup_dir, instance=None, backup_id=None, options=[]): cmd_list = [ "delete", - "-B", self.backup_dir(node), + "-B", backup_dir ] - if id: - cmd_list += ["-i", id] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] # print(cmd_list) return self.run_pb(cmd_list + options) - def delete_expired(self, node, options=[]): + def delete_expired(self, backup_dir, instance, options=[]): cmd_list = [ "delete", "--expired", - "-B", self.backup_dir(node), + "-B", backup_dir, + "--instance={0}".format(instance) ] return self.run_pb(cmd_list + options) - def show_config(self, node): + def show_config(self, backup_dir, instance): out_dict = {} cmd_list = [ "show-config", - "-B", self.backup_dir(node), + "-B", backup_dir, + "--instance={0}".format(instance) ] res = self.run_pb(cmd_list).splitlines() for line in res: @@ -485,9 +510,7 @@ class ProbackupTest(object): out_dict[key.strip()] = value.strip(" '").replace("'\n", "") return out_dict - def set_archiving_conf(self, node, archive_dir=False, replica=False): - if not archive_dir: - archive_dir = self.arcwal_dir(node) + def set_archiving(self, backup_dir, instance, node, replica=False): if replica: archive_mode = 'always' @@ -506,8 +529,8 @@ class ProbackupTest(object): if os.name == 'posix': node.append_conf( "postgresql.auto.conf", - "archive_command = 'test ! -f {0}/%f && cp %p {0}/%f'".format(archive_dir) - ) + "archive_command = '{0} archive-push -B {1} --instance={2} --wal-file-path %p --wal-file-name %f'".format( + self.probackup_path, backup_dir, instance)) #elif os.name == 'nt': # node.append_conf( # "postgresql.auto.conf", @@ -536,3 +559,7 @@ class ProbackupTest(object): return str(var[0][0]) else: return False + + def get_username(self): + """ Returns current user name """ + return pwd.getpwuid(os.getuid())[0] diff --git a/tests/init_test.py b/tests/init_test.py index 173c8ffd..b19f069d 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -3,56 +3,70 @@ from sys import exit import os from os import path import six -from .ptrack_helpers import dir_files, ProbackupTest, ProbackupException +from helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException -#TODO class InitTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(InitTest, self).__init__(*args, **kwargs) + self.module_name = 'init' - def test_success_1(self): + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_success(self): """Success normal init""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname)) - self.assertEqual(self.init_pb(node), six.b("")) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname)) + self.init_pb(backup_dir) self.assertEqual( - dir_files(self.backup_dir(node)), - ['backups', 'pg_probackup.conf', 'wal'] + dir_files(backup_dir), + ['backups', 'wal'] ) + self.add_instance(backup_dir, 'node', node) - def test_already_exist_2(self): + self.assertEqual("INFO: Instance 'node' successfully deleted\n", + self.del_instance(backup_dir, 'node', node), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + def test_already_exist(self): """Failure with backup catalog already existed""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname)) - self.init_pb(node) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname)) + self.init_pb(backup_dir) try: - self.init_pb(node) - # we should die here because exception is what we expect to happen - exit(1) + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - "ERROR: backup catalog already exist and it's not empty\n" - ) + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - def test_abs_path_3(self): + def test_abs_path(self): """failure with backup catalog should be given as absolute path""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname)) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname)) try: self.run_pb(["init", "-B", path.relpath("%s/backup" % node.base_dir, self.dir_path)]) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - "ERROR: -B, --backup-path must be an absolute path\n" - ) + self.assertEqual(e.message, + "ERROR: -B, --backup-path must be an absolute path\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) if __name__ == '__main__': diff --git a/tests/option_test.py b/tests/option_test.py index d16d4fe0..1114c169 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -1,7 +1,7 @@ import unittest -from os import path +import os import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from testgres import stop_all @@ -9,178 +9,206 @@ class OptionTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(OptionTest, self).__init__(*args, **kwargs) + self.module_name = 'option' @classmethod def tearDownClass(cls): stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_help_1(self): """help options""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - with open(path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: self.assertEqual( self.run_pb(["--help"]), help_out.read() ) + # @unittest.skip("skip") def test_version_2(self): """help options""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - with open(path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: self.assertEqual( self.run_pb(["--version"]), version_out.read() ) + # @unittest.skip("skip") def test_without_backup_path_3(self): """backup command failure without backup mode option""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') try: self.run_pb(["backup", "-b", "full"]) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n' - ) + self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # @unittest.skip("skip") def test_options_4(self): """check options test""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/option/{0}".format(fname)) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}) + try: node.stop() except: pass - self.assertEqual(self.init_pb(node), six.b("")) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # backup command failure without instance option + try: + self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + 'ERROR: required parameter not specified: --instance\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # backup command failure without backup mode option try: - self.run_pb(["backup", "-B", self.backup_dir(node), "-D", node.data_dir]) - # we should die here because exception is what we expect to happen - exit(1) + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) + self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: -# print e.message - self.assertEqual( - e.message, - 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n' - ) + self.assertEqual(e.message, + 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # backup command failure with invalid backup mode option try: - self.run_pb(["backup", "-b", "bad", "-B", self.backup_dir(node)]) - # we should die here because exception is what we expect to happen - exit(1) + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) + self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: invalid backup-mode "bad"\n' - ) + self.assertEqual(e.message, + 'ERROR: invalid backup-mode "bad"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # delete failure without ID try: - self.run_pb(["delete", "-B", self.backup_dir(node)]) + self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: required backup ID not specified\n' - ) + self.assertEqual(e.message, + 'ERROR: required backup ID not specified\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + #@unittest.skip("skip") + def test_options_5(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}) + + self.assertEqual(self.init_pb(backup_dir), six.b("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir))) + self.add_instance(backup_dir, 'node', node) node.start() # syntax error in pg_probackup.conf - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write(" = INFINITE\n") - try: - self.backup_pb(node) + self.backup_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: syntax error in " = INFINITE"\n' - ) + self.assertEqual(e.message, + 'ERROR: syntax error in " = INFINITE"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.clean_pb(node) - self.assertEqual(self.init_pb(node), six.b("")) + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) # invalid value in pg_probackup.conf - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("BACKUP_MODE=\n") try: - self.backup_pb(node, backup_type=None), + self.backup_node(backup_dir, 'node', node, backup_type=None), # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: invalid backup-mode ""\n' - ) + self.assertEqual(e.message, + 'ERROR: invalid backup-mode ""\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.clean_pb(node) + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) # Command line parameters should override file values - self.assertEqual(self.init_pb(node), six.b("")) - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy=1\n") self.assertEqual( - self.show_config(node)['retention-redundancy'], + self.show_config(backup_dir, 'node')['retention-redundancy'], six.b('1') ) # User cannot send --system-identifier parameter via command line try: - self.backup_pb(node, options=["--system-identifier", "123"]), + self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: option system-identifier cannot be specified in command line\n' - ) + self.assertEqual(e.message, + 'ERROR: option system-identifier cannot be specified in command line\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # invalid value in pg_probackup.conf - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("SMOOTH_CHECKPOINT=FOO\n") try: - self.backup_pb(node), + self.backup_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n" - ) + self.assertEqual(e.message, + "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.clean_pb(node) - self.assertEqual(self.init_pb(node), six.b("")) + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) # invalid option in pg_probackup.conf - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("TIMELINEID=1\n") try: - self.backup_pb(node), + self.backup_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: invalid option "TIMELINEID"\n' - ) + self.assertEqual(e.message, + 'ERROR: invalid option "TIMELINEID"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.clean_pb(node) - self.assertEqual(self.init_pb(node), six.b("")) - - node.stop() +# self.clean_pb(backup_dir) +# node.stop() diff --git a/tests/pgpro560.py b/tests/pgpro560.py new file mode 100644 index 00000000..b0117f70 --- /dev/null +++ b/tests/pgpro560.py @@ -0,0 +1,78 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class CheckSystemID(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(CheckSystemID, self).__init__(*args, **kwargs) + + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro560_control_file_loss(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node with stream support, delete control file + make backup + check that backup failed + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + file = os.path.join(node.base_dir,'data', 'global', 'pg_control') + os.remove(file) + + try: + self.backup_pb(node, backup_type='full', options=['--stream']) + assertEqual(1, 0, 'Error is expected because of control file loss') + except ProbackupException, e: + self.assertTrue( + 'ERROR: could not open file' and 'pg_control' in e.message, + 'Expected error is about control file loss') + + def test_pgpro560_systemid_mismatch(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node1 and node2 + feed to backup PGDATA from node1 and PGPORT from node2 + check that backup failed + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node1".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node1.start() + node2 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node2".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node2.start() + self.assertEqual(self.init_pb(node1), six.b("")) + + try: + self.backup_pb(node1, data_dir=node2.data_dir, backup_type='full', options=['--stream']) + assertEqual(1, 0, 'Error is expected because of SYSTEM ID mismatch') + except ProbackupException, e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' and + 'but target backup directory system id is' in e.message, + 'Expected error is about SYSTEM ID mismatch') diff --git a/tests/pgpro589.py b/tests/pgpro589.py new file mode 100644 index 00000000..00988d05 --- /dev/null +++ b/tests/pgpro589.py @@ -0,0 +1,97 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class ArchiveCheck(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ArchiveCheck, self).__init__(*args, **kwargs) + + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_mode(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check ERROR text + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() + + self.assertEqual(self.init_pb(node), six.b("")) + + try: + self.backup_pb(node, backup_type='full', options=['--archive-timeout=10']) + assertEqual(1, 0, 'Error is expected because of disabled archive_mode') + except ProbackupException, e: + self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n') + + def test_pgpro589(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check that backup status equal to ERROR + check that no files where copied to backup catalogue + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node.append_conf("postgresql.auto.conf", "archive_mode = on") + node.append_conf("postgresql.auto.conf", "wal_level = archive") + node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() + self.assertEqual(self.init_pb(node), six.b("")) + + try: + self.backup_pb( + node, backup_type='full', options=['--archive-timeout=10']) + assertEqual(1, 0, 'Error is expected because of missing archive wal segment with start_backup() LSN') + except ProbackupException, e: + self.assertTrue('INFO: wait for LSN' in e.message, "Expecting 'INFO: wait for LSN'") + self.assertTrue('ERROR: switched WAL segment' and 'could not be archived' in e.message, + "Expecting 'ERROR: switched WAL segment could not be archived'") + + id = self.show_pb(node)[0]['ID'] + self.assertEqual('ERROR', self.show_pb(node, id=id)['status'], 'Backup should have ERROR status') + #print self.backup_dir(node) + file = os.path.join(self.backup_dir(node), 'backups', id, 'database', path) + self.assertFalse(os.path.isfile(file), + '\n Start LSN was not found in archive but datafiles where copied to backup catalogue.\n For example: {0}\n It is not optimal'.format(file)) diff --git a/tests/pgpro688.py b/tests/pgpro688.py new file mode 100644 index 00000000..416d2cf5 --- /dev/null +++ b/tests/pgpro688.py @@ -0,0 +1,201 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import stop_all, get_username +import subprocess +from sys import exit, _getframe +import shutil +import time + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ReplicaTest, self).__init__(*args, **kwargs) + self.module_name = 'replica' + self.instance_master = 'master' + self.instance_replica = 'replica' + +# @classmethod +# def tearDownClass(cls): +# stop_all() + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_full_backup(self): + """make full stream backup from replica""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, self.instance_master, master) + master.start() + + # Make empty Object 'replica' from new node + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname)) + replica_port = replica.port + replica.cleanup() + + # FULL STREAM backup of master + self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream']) + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + before = master.execute("postgres", "SELECT * FROM t_heap") + + # FULL STREAM backup of master + self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream']) + + # Restore last backup from master to Replica directory + self.restore_node(backup_dir, self.instance_master, replica.data_dir) + # Set Replica + replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf('recovery.conf', + "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port)) + replica.start({"-t": "600"}) + + # Check replica + after = replica.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Add instance replica + self.add_instance(backup_dir, self.instance_replica, replica) + + # FULL STREAM backup of replica + self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in + self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[ + '--stream', '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) + + # Validate instance replica + self.validate_pb(backup_dir, self.instance_replica) + self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) + + def test_replica_archive_full_backup(self): + """make page archive backup from replica""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'} + ) + self.set_archiving(backup_dir, self.instance_master, master) + self.init_pb(backup_dir) + self.add_instance(backup_dir, self.instance_master, master) + master.start() + + # Make empty Object 'replica' from new node + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname)) + replica_port = replica.port + replica.cleanup() + + # FULL ARCHIVE backup of master + self.backup_node(backup_dir, self.instance_master, master, backup_type='full') + # Create table t_heap + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + before = master.execute("postgres", "SELECT * FROM t_heap") + + # PAGE ARCHIVE backup of master + self.backup_node(backup_dir, self.instance_master, master, backup_type='page') + + # Restore last backup from master to Replica directory + self.restore_node(backup_dir, self.instance_master, replica.data_dir) + + # Set Replica + self.set_archiving(backup_dir, self.instance_replica, replica, replica=True) + replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf('recovery.conf', + "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port)) + replica.start({"-t": "600"}) + + # Check replica + after = replica.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Make FULL ARCHIVE backup from replica + self.add_instance(backup_dir, self.instance_replica, replica) + self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in + self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[ + '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) + self.validate_pb(backup_dir, self.instance_replica) + self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) + + # Drop Table t_heap + after = master.execute("postgres", "drop table t_heap") + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,512) i") + before = master.execute("postgres", "SELECT * FROM t_heap") + + # Make page backup from replica + self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in + self.backup_node(backup_dir, self.instance_replica, replica, backup_type='page', options=[ + '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) + self.validate_pb(backup_dir, self.instance_replica) + self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) + + @unittest.skip("skip") + def test_replica_archive_full_backup_123(self): + """ + make full archive backup from replica + """ + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/master".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.start() + + replica = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/replica".format(fname)) + replica_port = replica.port + replica.cleanup() + + self.assertEqual(self.init_pb(master), six.b("")) + self.backup_pb(node=master, backup_type='full', options=['--stream']) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + + before = master.execute("postgres", "SELECT * FROM t_heap") + + id = self.backup_pb(master, backup_type='page', options=['--stream']) + self.restore_pb(backup_dir=self.backup_dir(master), data_dir=replica.data_dir) + + # Settings for Replica + replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + # Set Archiving for replica + self.set_archiving_conf(replica, replica=True) + + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf('recovery.conf', + "primary_conninfo = 'user=gsmol port={0} sslmode=prefer sslcompression=1'".format(master.port)) + replica.start({"-t": "600"}) + # Replica Started + + # master.execute("postgres", "checkpoint") + + # Check replica + after = replica.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Make backup from replica + self.assertEqual(self.init_pb(replica), six.b("")) + self.backup_pb(replica, backup_type='full', options=['--archive-timeout=30']) + self.validate_pb(replica) diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 597ea7e7..0880c031 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -1,12 +1,14 @@ import unittest +import os from sys import exit from testgres import get_new_node, stop_all -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_clean' def teardown(self): stop_all() @@ -15,13 +17,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure def test_ptrack_clean(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - set_archiving=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -35,8 +40,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) # Make full backup to clean every ptrack - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) for i in idx_ptrack: # get fork size and calculate it in pages @@ -52,7 +56,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): node.psql('postgres', 'update t_heap set text = md5(text), tsvector = md5(repeat(tsvector::text, 10))::tsvector;') node.psql('postgres', 'vacuum t_heap') - id = self.backup_pb(node, backup_type='ptrack', options=['-j100', '--stream']) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['-j100', '--stream']) node.psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -71,7 +75,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): node.psql('postgres', 'vacuum t_heap') # Make page backup to clean every ptrack - self.backup_pb(node, backup_type='page', options=['-j100']) + self.backup_node(backup_dir, 'node', node, backup_type='page', options=['-j100']) node.psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -85,8 +89,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - print self.show_pb(node, as_text=True) - self.clean_pb(node) + print self.show_pb(backup_dir, 'node', as_text=True) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index e4525bfd..ff0fb6a2 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -1,26 +1,32 @@ import unittest +import os from sys import exit from testgres import get_new_node, stop_all -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_cluster' def teardown(self): # clean_all() stop_all() - # @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_cluster_btree(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -45,8 +51,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_btree') @@ -67,18 +72,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() - # @unittest.skip("123") def test_ptrack_cluster_spgist(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -103,8 +109,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_spgist') @@ -125,18 +130,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() - # @unittest.skip("123") def test_ptrack_cluster_brin(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -161,8 +167,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_brin') @@ -183,18 +188,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() - # @unittest.skip("123") def test_ptrack_cluster_gist(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -219,8 +225,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_gist') @@ -241,18 +246,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() - # @unittest.skip("123") def test_ptrack_cluster_gin(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -277,8 +283,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_gin') @@ -299,7 +304,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index d43282f1..e35234a6 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -3,26 +3,32 @@ from sys import exit from testgres import get_new_node, stop_all import os from signal import SIGTERM -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack from time import sleep class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_move_to_tablespace' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_recovery(self): - fname = self.id().split(".")[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -52,7 +58,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index 73e9e085..24802697 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -3,26 +3,32 @@ from sys import exit from testgres import get_new_node, stop_all import os from signal import SIGTERM -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack from time import sleep class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_move_to_tablespace' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_recovery(self): - fname = self.id().split(".")[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table @@ -54,7 +60,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index 484c5c50..f6b22b97 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -1,12 +1,14 @@ import unittest +import os from sys import exit from testgres import get_new_node, stop_all -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum' def teardown(self): # clean_all() @@ -16,12 +18,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure def test_ptrack_vacuum(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -47,8 +52,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) @@ -74,7 +78,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index 75d25909..1a4d3fe5 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -1,25 +1,32 @@ +import os import unittest from sys import exit from testgres import get_new_node, stop_all -from os import path, open, lseek, read, close, O_RDONLY -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum_bits_frozen' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum_bits_frozen(self): - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_bits_frozen", + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -43,8 +50,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'vacuum freeze t_heap') node.psql('postgres', 'checkpoint') @@ -63,8 +69,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py index 4fc12419..ca5db705 100644 --- a/tests/ptrack_vacuum_bits_visibility.py +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -1,25 +1,32 @@ +import os import unittest from sys import exit from testgres import get_new_node, stop_all -from os import path, open, lseek, read, close, O_RDONLY -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum_bits_visibility' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum_bits_visibility(self): - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_bits_visibility", + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -43,8 +50,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'vacuum t_heap') node.psql('postgres', 'checkpoint') @@ -63,8 +69,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index 98af70be..9d9d5051 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -1,38 +1,33 @@ +import os import unittest from sys import exit from testgres import get_new_node, stop_all #import os -from os import path, open, lseek, read, close, O_RDONLY -from .ptrack_helpers import ProbackupTest, idx_ptrack - -# res = node.execute('postgres', 'show fsync') -# print res[0][0] -# res = node.execute('postgres', 'show wal_level') -# print res[0][0] -# a = ProbackupTest -# res = node.execute('postgres', 'select 1')` -# self.assertEqual(len(res), 1) -# self.assertEqual(res[0][0], 1) -# node.stop() -# a = self.backup_dir(node) +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum_full' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum_full(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -57,8 +52,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'vacuum full t_heap') @@ -78,8 +72,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity, the most important part self.check_ptrack_sanity(idx_ptrack[i]) - - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index eba15da4..37dd9920 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -1,25 +1,32 @@ +import os import unittest from sys import exit from testgres import get_new_node, stop_all -from os import path, open, lseek, read, close, O_RDONLY -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum_truncate' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum_truncate(self): - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_truncate", + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -44,8 +51,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id > 128;') node.psql('postgres', 'vacuum t_heap') @@ -65,8 +71,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/replica.py b/tests/replica.py new file mode 100644 index 00000000..9f3e90f8 --- /dev/null +++ b/tests/replica.py @@ -0,0 +1,129 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit, _getframe +import shutil +import time + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ReplicaTest, self).__init__(*args, **kwargs) + self.module_name = 'replica' + +# @classmethod +# def tearDownClass(cls): +# stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_full_backup(self): + """make full stream backup from replica""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} + ) + master.start() + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + slave = self.make_simple_node(base_dir="{0}/{1}/slave".format(self.module_name, fname)) + slave.cleanup() + + # FULL BACKUP + self.backup_node(backup_dir, 'master', master, backup_type='full', options=['--stream']) + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + + before = master.execute("postgres", "SELECT * FROM t_heap") + + #FULL BACKUP + self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.restore_node(backup_dir, 'master', slave) + + slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port)) + slave.append_conf('postgresql.auto.conf', 'hot_standby = on') + + slave.append_conf('recovery.conf', "standby_mode = 'on'") + slave.append_conf('recovery.conf', + "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(self.user, master.port)) + slave.start({"-t": "600"}) + # Replica Ready + + # Check replica + after = slave.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Make backup from replica + self.add_instance(backup_dir, 'slave', slave) + #time.sleep(2) + self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in + self.backup_node(backup_dir, 'slave', slave, options=['--stream', '--log-level=verbose', + '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)])) + self.validate_pb(backup_dir, 'slave') + self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status']) + + # @unittest.skip("skip") + def test_replica_archive_full_backup(self): + """make full archive backup from replica""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.start() + + slave = self.make_simple_node(base_dir="{0}/{1}/slave".format(self.module_name, fname)) + slave.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + + before = master.execute("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node(backup_dir, 'master', master, backup_type='page') + self.restore_node(backup_dir, 'master', slave) + + # Settings for Replica + slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port)) + slave.append_conf('postgresql.auto.conf', 'hot_standby = on') + # Set Archiving for replica + #self.set_archiving_conf( slave, replica=True) + self.set_archiving(backup_dir, 'slave', slave, replica=True) + + # Set Replica + slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port)) + slave.append_conf('postgresql.auto.conf', 'hot_standby = on') + slave.append_conf('recovery.conf', "standby_mode = 'on'") + slave.append_conf('recovery.conf', + "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(self.user, master.port)) + slave.start({"-t": "600"}) + + # Check replica + after = slave.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Make backup from replica + self.add_instance(backup_dir, 'slave', slave) + self.backup_node(backup_dir, 'slave', slave, options=['--archive-timeout=300', + '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'slave') diff --git a/tests/restore_test.py b/tests/restore_test.py index ea35a0b1..1ef35c7d 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1,54 +1,57 @@ import unittest import os -from os import path import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from testgres import stop_all import subprocess from datetime import datetime import shutil +from sys import exit class RestoreTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(RestoreTest, self).__init__(*args, **kwargs) + self.module_name = 'restore' @classmethod def tearDownClass(cls): stop_all() -# @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_restore_full_to_latest(self): """recovery to latest from full backup""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() before = node.execute("postgres", "SELECT * FROM pgbench_branches") - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) node.stop({"-m": "immediate"}) node.cleanup() # 1 - Test recovery from latest -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) # 2 - Test that recovery.conf was created - recovery_conf = path.join(node.data_dir, "recovery.conf") - self.assertEqual(path.isfile(recovery_conf), True) + recovery_conf = os.path.join(node.data_dir, "recovery.conf") + self.assertEqual(os.path.isfile(recovery_conf), True) node.start({"-t": "600"}) @@ -57,37 +60,38 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() + # @unittest.skip("skip") def test_restore_full_page_to_latest(self): """recovery to latest from full + page backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") before = node.execute("postgres", "SELECT * FROM pgbench_branches") node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -96,31 +100,33 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() + # @unittest.skip("skip") def test_restore_to_timeline(self): """recovery to target timeline""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) before = node.execute("postgres", "SELECT * FROM pgbench_branches") - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) target_tli = int(node.get_control_data()[six.b("Latest checkpoint's TimeLineID")]) node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -128,17 +134,15 @@ class RestoreTest(ProbackupTest, unittest.TestCase): pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node, backup_type="full") node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, - options=["-j", "4", "--verbose", "--timeline=%i" % target_tli]) -# ) + # Correct Backup must be choosen for restore + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", "--timeline={0}".format(target_tli)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] self.assertEqual(int(recovery_target_timeline), target_tli) @@ -150,36 +154,36 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() + # @unittest.skip("skip") def test_restore_to_time(self): - """recovery to target timeline""" + """recovery to target time""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - node.pgbench_init(scale=2) + node.pgbench_init(scale=2) before = node.execute("postgres", "SELECT * FROM pgbench_branches") - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - node.stop({"-m": "immediate"}) + node.stop() node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, - options=["-j", "4", "--verbose", '--time="%s"' % target_time]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(target_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -188,23 +192,26 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() + # @unittest.skip("skip") def test_restore_to_xid(self): """recovery to target xid""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() @@ -223,16 +230,14 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Enforce segment to be archived to ensure that recovery goes up to the # wanted point. There is no way to ensure that all segments needed have # been archived up to the xmin point saved earlier without that. - node.execute("postgres", "SELECT pg_switch_xlog()") + #node.execute("postgres", "SELECT pg_switch_xlog()") node.stop({"-m": "fast"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, - options=["-j", "4", "--verbose", '--xid=%s' % target_xid]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--xid={0}'.format(target_xid)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -241,45 +246,38 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() - def test_restore_full_ptrack(self): - """recovery to latest from full + ptrack backups""" + # @unittest.skip("skip") + def test_restore_full_ptrack_archive(self): + """recovery to latest from archive full+ptrack backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - node.append_conf("postgresql.conf", "ptrack_enable = on") - node.restart() - - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack") before = node.execute("postgres", "SELECT * FROM pgbench_branches") node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -288,52 +286,44 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() - def test_restore_full_ptrack_ptrack(self): - """recovery to latest from full + ptrack + ptrack backups""" + # @unittest.skip("skip") + def test_restore_ptrack(self): + """recovery to latest from archive full+ptrack+ptrack backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - node.append_conf("postgresql.conf", "ptrack_enable = on") - node.restart() - - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack") before = node.execute("postgres", "SELECT * FROM pgbench_branches") node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -342,42 +332,39 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() + # @unittest.skip("skip") def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose", "--stream"])) + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node, options=["--stream"]) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"]) before = node.execute("postgres", "SELECT * FROM pgbench_branches") - node.stop({"-m": "immediate"}) + node.stop() node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -386,28 +373,24 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() + # @unittest.skip("skip") def test_restore_full_ptrack_under_load(self): """recovery to latest from full + ptrack backups with loads when ptrack backup do""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - wal_segment_size = self.guc_wal_segment_size(node) - node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - node.restart() - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -415,8 +398,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): options=["-c", "4", "-T", "8"] ) - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"]) pgbench.wait() pgbench.stdout.close() @@ -428,12 +410,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop({"-m": "immediate"}) node.cleanup() - self.wrong_wal_clean(node, wal_segment_size) - -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -444,26 +423,23 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() + # @unittest.skip("skip") def test_restore_full_under_load_ptrack(self): """recovery to latest from full + page backups with loads when full backup do""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - wal_segment_size = self.guc_wal_segment_size(node) - node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - node.restart() + # wal_segment_size = self.guc_wal_segment_size(node) + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -471,14 +447,12 @@ class RestoreTest(ProbackupTest, unittest.TestCase): options=["-c", "4", "-T", "8"] ) - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"]) bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history") @@ -487,39 +461,39 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop({"-m": "immediate"}) node.cleanup() - self.wrong_wal_clean(node, wal_segment_size) - -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + #self.wrong_wal_clean(node, wal_segment_size) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history") self.assertEqual(bbalance, delta) - node.stop() + # @unittest.skip("skip") def test_restore_to_xid_inclusive(self): """recovery with target inclusive false""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() @@ -538,20 +512,15 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Enforce segment to be archived to ensure that recovery goes up to the # wanted point. There is no way to ensure that all segments needed have # been archived up to the xmin point saved earlier without that. - node.execute("postgres", "SELECT pg_switch_xlog()") + # node.execute("postgres", "SELECT pg_switch_xlog()") node.stop({"-m": "fast"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, - options=[ - "-j", "4", - "--verbose", - '--xid=%s' % target_xid, - "--inclusive=false"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, + options=["-j", "4", '--xid={0}'.format(target_xid), "--inclusive=false"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -561,19 +530,22 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() + # @unittest.skip("skip") def test_restore_with_tablespace_mapping_1(self): """recovery using tablespace-mapping option""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) # Create tablespace - tblspc_path = path.join(node.base_dir, "tblspc") + tblspc_path = os.path.join(node.base_dir, "tblspc") os.makedirs(tblspc_path) with node.connect("postgres") as con: con.connection.autocommit = True @@ -583,88 +555,88 @@ class RestoreTest(ProbackupTest, unittest.TestCase): con.execute("INSERT INTO test VALUES (1)") con.commit() - self.backup_pb(node) - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + backup_id = self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) # 1 - Try to restore to existing directory node.stop() try: - self.restore_pb(node) + self.restore_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because restore destionation is not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir) - ) + self.assertEqual(e.message, + 'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # 2 - Try to restore to existing tablespace directory - shutil.rmtree(node.data_dir) + node.cleanup() try: - self.restore_pb(node) + self.restore_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because restore tablespace destination is not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path) - ) + self.assertEqual(e.message, + 'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # 3 - Restore using tablespace-mapping - tblspc_path_new = path.join(node.base_dir, "tblspc_new") -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete."), - self.restore_pb(node, - options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]) -# ) + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() - id = node.execute("postgres", "SELECT id FROM test") - self.assertEqual(id[0][0], 1) + res = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(res[0][0], 1) # 4 - Restore using tablespace-mapping using page backup - self.backup_pb(node) + self.backup_node(backup_dir, 'node', node) with node.connect("postgres") as con: con.execute("INSERT INTO test VALUES (2)") con.commit() - self.backup_pb(node, backup_type="page") + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") - show_pb = self.show_pb(node) + show_pb = self.show_pb(backup_dir, 'node') self.assertEqual(show_pb[1]['Status'], six.b("OK")) self.assertEqual(show_pb[2]['Status'], six.b("OK")) node.stop() node.cleanup() - tblspc_path_page = path.join(node.base_dir, "tblspc_page") -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete."), - self.restore_pb(node, - options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]) -# ) + tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") + + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() - id = node.execute("postgres", "SELECT id FROM test OFFSET 1") - self.assertEqual(id[0][0], 2) + res = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(res[0][0], 2) node.stop() + # @unittest.skip("skip") def test_restore_with_tablespace_mapping_2(self): """recovery using tablespace-mapping option and page backup""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) # Full backup - self.backup_pb(node) - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) # Create tablespace - tblspc_path = path.join(node.base_dir, "tblspc") + tblspc_path = os.path.join(node.base_dir, "tblspc") os.makedirs(tblspc_path) with node.connect("postgres") as con: con.connection.autocommit = True @@ -674,9 +646,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): con.commit() # First page backup - self.backup_pb(node, backup_type="page") - self.assertEqual(self.show_pb(node)[1]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(node)[1]['Mode'], six.b("PAGE")) + self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], six.b("PAGE")) # Create tablespace table with node.connect("postgres") as con: @@ -688,28 +660,119 @@ class RestoreTest(ProbackupTest, unittest.TestCase): con.commit() # Second page backup - self.backup_pb(node, backup_type="page") - self.assertEqual(self.show_pb(node)[2]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(node)[2]['Mode'], six.b("PAGE")) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], six.b("PAGE")) node.stop() node.cleanup() - tblspc_path_new = path.join(node.base_dir, "tblspc_new") -# exit(1) -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete."), - self.restore_pb(node, - options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]) -# ) + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - # Check tables + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() count = node.execute("postgres", "SELECT count(*) FROM tbl") self.assertEqual(count[0][0], 4) - count = node.execute("postgres", "SELECT count(*) FROM tbl1") self.assertEqual(count[0][0], 4) - + node.stop() + + # @unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """make node with archiving, make stream backup, make PITR to Recovery Time""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + node.psql("postgres", "create table t_heap(a int)") + node.psql("postgres", "select pg_switch_xlog()") + node.stop() + node.cleanup() + + recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + node.start({"-t": "600"}) + + res = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in res[2]) + self.assertEqual(True, node.status()) + node.stop() + + # @unittest.skip("skip") + def test_archive_node_backup_stream_pitr(self): + """make node with archiving, make stream backup, create table t_heap, make pitr to Recovery Time, check that t_heap do not exists""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + node.psql("postgres", "create table t_heap(a int)") + node.cleanup() + + recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, + options=["-j", "4", '--time="{0}"'.format(recovery_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + node.start({"-t": "600"}) + + res = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in res[2]) + node.stop() + + # @unittest.skip("skip") + def test_archive_node_backup_archive_pitr_2(self): + """make node with archiving, make archive backup, create table t_heap, make pitr to Recovery Time, check that t_heap do not exists""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + node.psql("postgres", "create table t_heap(a int)") + node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) + node.cleanup() + + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, + options=["-j", "4", '--time="{0}"'.format(recovery_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + node.start({"-t": "600"}) + + res = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in res[2]) node.stop() diff --git a/tests/retention_test.py b/tests/retention_test.py index 265ed8da..8be47b6f 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -2,7 +2,7 @@ import unittest import os from datetime import datetime, timedelta from os import path, listdir -from .ptrack_helpers import ProbackupTest +from helpers.ptrack_helpers import ProbackupTest from testgres import stop_all @@ -10,39 +10,42 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(RetentionTest, self).__init__(*args, **kwargs) + self.module_name = 'retention' @classmethod def tearDownClass(cls): stop_all() -# @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.init_pb(node) - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy = 1\n") # Make backups to be purged - self.backup_pb(node) - self.backup_pb(node, backup_type="page") + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") # Make backups to be keeped - self.backup_pb(node) - self.backup_pb(node, backup_type="page") + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(len(self.show_pb(node)), 4) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) # Purge backups - log = self.delete_expired(node) - self.assertEqual(len(self.show_pb(node)), 2) + log = self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) # Check that WAL segments were deleted min_wal = None @@ -52,7 +55,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): min_wal = line[31:-1] elif line.startswith(b"INFO: removed max WAL segment"): max_wal = line[31:-1] - for wal_name in listdir(path.join(self.backup_dir(node), "wal")): + for wal_name in listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): wal_name_b = wal_name.encode('ascii') self.assertEqual(wal_name_b[8:] > min_wal[8:], True) @@ -64,40 +67,43 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.init_pb(node) - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy = 1\n") conf.write("retention-window = 1\n") # Make backups to be purged - self.backup_pb(node) - self.backup_pb(node, backup_type="page") + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") # Make backup to be keeped - self.backup_pb(node) + self.backup_node(backup_dir, 'node', node) - backups = path.join(self.backup_dir(node), "backups") + backups = path.join(backup_dir, 'backups', 'node') days_delta = 5 for backup in listdir(backups): + if backup == 'pg_probackup.conf': + continue with open(path.join(backups, backup, "backup.control"), "a") as conf: conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=days_delta))) days_delta -= 1 # Make backup to be keeped - self.backup_pb(node, backup_type="page") + self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(len(self.show_pb(node)), 4) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) # Purge backups - self.delete_expired(node) - self.assertEqual(len(self.show_pb(node)), 2) + self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) node.stop() diff --git a/tests/show_test.py b/tests/show_test.py index d5fdc9fc..37b146e4 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -2,7 +2,7 @@ import unittest import os from os import path import six -from .ptrack_helpers import ProbackupTest +from helpers.ptrack_helpers import ProbackupTest from testgres import stop_all @@ -10,46 +10,56 @@ class OptionTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(OptionTest, self).__init__(*args, **kwargs) + self.module_name = 'show' @classmethod def tearDownClass(cls): stop_all() - def show_test_1(self): + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_1(self): """Status DONE and OK""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/show/{0}".format(fname), - set_archiving=True, + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) self.assertEqual( - self.backup_pb(node, options=["--quiet"]), + self.backup_node(backup_dir, 'node', node, options=["--log-level=panic"]), None ) - self.assertIn(six.b("OK"), self.show_pb(node, as_text=True)) + self.assertIn(six.b("OK"), self.show_pb(backup_dir, 'node', as_text=True)) node.stop() + # @unittest.skip("skip") def test_corrupt_2(self): """Status CORRUPT""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/show/{0}".format(fname), - set_archiving=True, + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - id_backup = self.backup_pb(node) - path.join(self.backup_dir(node), "backups", id_backup.decode("utf-8"), "database", "postgresql.conf") - os.remove(path.join(self.backup_dir(node), "backups", id_backup.decode("utf-8"), "database", "postgresql.conf")) + backup_id = self.backup_node(backup_dir, 'node', node) - self.validate_pb(node, id_backup) - self.assertIn(six.b("CORRUPT"), self.show_pb(node, as_text=True)) + # delete file which belong to backup + file = path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + self.validate_pb(backup_dir, 'node', backup_id) + self.assertIn(six.b("CORRUPT"), self.show_pb(backup_dir, as_text=True)) node.stop() diff --git a/tests/validate_test.py b/tests/validate_test.py index f3fdf966..bef73cd7 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1,41 +1,47 @@ import unittest import os import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta from testgres import stop_all import subprocess +from sys import exit +import re class ValidateTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(ValidateTest, self).__init__(*args, **kwargs) + self.module_name = 'validate' @classmethod def tearDownClass(cls): stop_all() -# @unittest.skip("123") - def test_validate_wal_1(self): - """recovery to latest from full backup""" + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_validate_wal_unreal_values(self): + """make node with archiving, make archive backup, validate to both real and unreal values""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - with open(os.path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -45,33 +51,33 @@ class ValidateTest(ProbackupTest, unittest.TestCase): pgbench.wait() pgbench.stdout.close() - id_backup = self.show_pb(node)[0]['ID'] - target_time = self.show_pb(node)[0]['Recovery time'] - after_backup_time = datetime.now() + target_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + after_backup_time = datetime.now().replace(second=0, microsecond=0) # Validate to real time - self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, options=["--time='{0}'".format(target_time)])) + self.assertIn(six.b("INFO: backup validation completed successfully"), + self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(target_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) # Validate to unreal time + unreal_time_1 = after_backup_time - timedelta(days=2) try: - self.validate_pb(node, options=["--time='{:%Y-%m-%d %H:%M:%S}'".format( - after_backup_time - timedelta(days=2))]) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal time") + self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_1)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n') + self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Validate to unreal time #2 + unreal_time_2 = after_backup_time + timedelta(days=2) try: - self.validate_pb(node, options=["--time='{:%Y-%m-%d %H:%M:%S}'".format( - after_backup_time + timedelta(days=2))]) - self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal time") + self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_2)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - True, - 'ERROR: not enough WAL records to time' in e.message - ) + self.assertTrue('ERROR: not enough WAL records to time' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Validate to real xid target_xid = None @@ -81,61 +87,132 @@ class ValidateTest(ProbackupTest, unittest.TestCase): target_xid = res[0][0] node.execute("postgres", "SELECT pg_switch_xlog()") - self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, options=["--xid=%s" % target_xid])) + self.assertIn(six.b("INFO: backup validation completed successfully"), + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) # Validate to unreal xid + unreal_xid = int(target_xid) + 1000 try: - self.validate_pb(node, options=["--xid=%d" % (int(target_xid) + 1000)]) - self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal xid") + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - True, - 'ERROR: not enough WAL records to xid' in e.message - ) + self.assertTrue('ERROR: not enough WAL records to xid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Validate with backup ID - self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, id_backup)) + self.assertIn(six.b("INFO: backup validation completed successfully"), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) - # Validate broken WAL - wals_dir = os.path.join(self.backup_dir(node), "wal") + # @unittest.skip("skip") + def test_validate_corrupt_wal_1(self): + """make archive node, make archive backup, corrupt all wal files, run validate, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals.sort() for wal in wals: f = open(os.path.join(wals_dir, wal), "rb+") - f.seek(256) + f.seek(42) + f.write(six.b("blablablaadssaaaaaaaaaaaaaaa")) + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertTrue('Possible WAL CORRUPTION' in e.message), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') + node.stop() + + # @unittest.skip("skip") + def test_validate_corrupt_wal_2(self): + """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + target_xid = None + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + f = open(os.path.join(wals_dir, wal), "rb+") + f.seek(0) f.write(six.b("blablabla")) f.close + # Validate to xid try: - self.validate_pb(node, id_backup, options=['--xid=%s' % target_xid]) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of wal segment corruption") + self.validate_pb(backup_dir, 'node', backup_id, options=['--xid={0}'.format(target_xid)]) + self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertTrue(True, 'Possible WAL CORRUPTION' in e.message) - - try: - self.validate_pb(node) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of wal segment corruption") - except ProbackupException, e: - self.assertTrue(True, 'Possible WAL CORRUPTION' in e.message) + self.assertTrue('Possible WAL CORRUPTION' in e.message), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd) + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') node.stop() -# @unittest.skip("123") + # @unittest.skip("skip") def test_validate_wal_lost_segment_1(self): - """Loose segment which belong to some backup""" + """make archive node, make archive full backup, + delete from archive wal segment which belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'CORRUPT' + """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -144,30 +221,56 @@ class ValidateTest(ProbackupTest, unittest.TestCase): ) pgbench.wait() pgbench.stdout.close() - self.backup_pb(node, backup_type='full') + backup_id = self.backup_node(backup_dir, 'node', node) - wals_dir = os.path.join(self.backup_dir(node), "wal") + # Delete wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - os.remove(os.path.join(self.backup_dir(node), "wal", wals[1])) + file = os.path.join(backup_dir, 'wal', 'node', wals[1]) + os.remove(file) try: - self.validate_pb(node) - self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance") + self.validate_pb(backup_dir, 'node') + self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertTrue('is absent' in e.message) + self.assertIn('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore'.format( + file), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"') + + # Be paranoid and run validate again + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) node.stop() - @unittest.expectedFailure + # @unittest.skip("skip") def test_validate_wal_lost_segment_2(self): - """Loose segment located between backups """ + """ + make node with archiving + make archive backup + delete from archive wal segment which DO NOT belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'ERROR' + """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + + self.backup_node(backup_dir, 'node', node) + + # make some wals node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -176,39 +279,24 @@ class ValidateTest(ProbackupTest, unittest.TestCase): ) pgbench.wait() pgbench.stdout.close() - self.backup_pb(node, backup_type='full') - - # need to do that to find segment between(!) backups - node.psql("postgres", "CREATE TABLE t1(a int)") - node.psql("postgres", "SELECT pg_switch_xlog()") - node.psql("postgres", "CREATE TABLE t2(a int)") - node.psql("postgres", "SELECT pg_switch_xlog()") - - wals_dir = os.path.join(self.backup_dir(node), "wal") - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals = map(int, wals) # delete last wal segment - os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))) + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = map(int, wals) + file = os.path.join(wals_dir, '0000000' + str(max(wals))) + os.remove(file) - # Need more accurate error message about loosing wal segment between backups try: - self.backup_pb(node, backup_type='page') - # we should die here because exception is what we expect to happen - exit(1) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) except ProbackupException, e: - self.assertEqual( - True, - 'could not read WAL record' in e.message - ) - self.delete_pb(node, id=self.show_pb(node)[1]['ID']) - - - ##### Hole Smokes, Batman! We just lost a wal segment and know nothing about it - ##### We need archive-push ASAP - self.backup_pb(node, backup_type='full') - self.assertEqual(False, - 'validation completed successfully' in self.validate_pb(node)) - ######## + self.assertTrue('INFO: wait for LSN' + and 'in archived WAL segment' + and 'WARNING: could not read WAL record at' + and 'ERROR: WAL segment "{0}" is absent\n'.format(file) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertEqual('ERROR', self.show_pb(backup_dir, 'node')[1]['Status'], 'Backup {0} should have STATUS "ERROR"') node.stop() From 133ec30346ae7fad3c435849fd5c7b6c383e1cc4 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 3 Jul 2017 18:14:35 +0300 Subject: [PATCH 04/37] Version 2.0 --- README.md | 184 --------------- src/archive.c | 3 + src/backup.c | 3 + src/data.c | 7 +- src/parsexlog.c | 2 +- src/pg_probackup.c | 9 +- src/pg_probackup.h | 1 + src/restore.c | 1 + src/validate.c | 3 + tests/__init__.py | 14 +- tests/backup_test.py | 90 ++++---- tests/delete_test.py | 33 ++- tests/expected/option_version.out | 2 +- tests/false_positive.py | 179 +++++++++------ tests/helpers/ptrack_helpers.py | 153 ++++++------- tests/init_test.py | 32 +-- tests/option_test.py | 49 ++-- tests/pb_lib.py | 304 ------------------------- tests/pgpro560.py | 60 +++-- tests/pgpro589.py | 76 ++++--- tests/pgpro688.py | 201 ---------------- tests/ptrack_clean.py | 16 +- tests/ptrack_cluster.py | 28 +-- tests/ptrack_move_to_tablespace.py | 18 +- tests/ptrack_recovery.py | 27 +-- tests/ptrack_vacuum.py | 16 +- tests/ptrack_vacuum_bits_frozen.py | 13 +- tests/ptrack_vacuum_bits_visibility.py | 13 +- tests/ptrack_vacuum_full.py | 14 +- tests/ptrack_vacuum_truncate.py | 13 +- tests/replica.py | 23 +- tests/restore_test.py | 195 ++++++++++------ tests/retention_test.py | 34 ++- tests/show_test.py | 25 +- tests/validate_test.py | 77 ++++--- 35 files changed, 645 insertions(+), 1273 deletions(-) delete mode 100644 README.md delete mode 100644 tests/pb_lib.py delete mode 100644 tests/pgpro688.py diff --git a/README.md b/README.md deleted file mode 100644 index 360fc81c..00000000 --- a/README.md +++ /dev/null @@ -1,184 +0,0 @@ -pg_probackup fork of pg_arman by Postgres Professional -======================================== - -pg_probackup is a backup and recovery manager for PostgreSQL servers able to do -differential and full backup as well as restore a cluster to a -state defined by a given recovery target. It is designed to perform -periodic backups of an existing PostgreSQL server, combined with WAL -archives to provide a way to recover a server in case of failure of -server because of a reason or another. Its differential backup -facility reduces the amount of data necessary to be taken between -two consecutive backups. - -Main features: -* incremental backup from WAL and PTRACK -* backup from replica -* multithreaded backup and restore -* autonomous backup without archive command (will need slot replication) - -Requirements: -* >=PostgreSQL 9.5 -* >=gcc 4.4 or >=clang 3.6 or >= XLC 12.1 -* pthread - -Download --------- - -The latest version of this software can be found on the project website at -https://github.com/postgrespro/pg_probackup. Original fork of pg_probackup can be -found at https://github.com/michaelpq/pg_arman. - -Installation ------------- - -Compiling pg_probackup requires a PostgreSQL installation to be in place -as well as a raw source tree. Pass the path to the PostgreSQL source tree -to make, in the top_srcdir variable: - - make USE_PGXS=1 top_srcdir= - -In addition, you must have pg_config in $PATH. - -The current version of pg_probackup is compatible with PostgreSQL 9.5 and -upper versions. - -Platforms ---------- - -pg_probackup has been tested on Linux and Unix-based platforms. - -Documentation -------------- - -All the documentation you can find [here](doc/pg_probackup.md). - -Regression tests ----------------- - -For tests you must have python 2.7 or python 3.3 and higher. Also good idea -is make virtual enviroment by `virtualenv`. -First of all you need to install `testgres` python module which contains useful -functions to start postgres clusters and make queries: - -``` -pip install testgres -``` - -To run tests execute: - -``` -python -m unittest tests -``` - -from current (root of project) directory. If you want to run a specific postgres build then -you should specify the path to your pg_config executable by setting PG_CONFIG -environment variable: -``` -export PG_CONFIG=/path/to/pg_config -``` - - -Block level incremental backup ------------------------------- - -Idea of block level incremental backup is that you may backup only blocks -changed since last full backup. It gives two major benefits: taking backups -faster and making backups smaller. - -The major question here is how to get the list of changed blocks. Since -each block contains LSN number, changed blocks could be retrieved by full scan -of all the blocks. But this approach consumes as much server IO as full -backup. - -This is why we implemented alternative approaches to retrieve -list of changed blocks. - -1. Scan WAL archive and extract changed blocks from it. However, shortcoming -of these approach is requirement to have WAL archive. - -2. Track bitmap of changes blocks inside PostgreSQL (ptrack). It introduces -some overhead to PostgreSQL performance. On our experiments it appears to be -less than 3%. - -These two approaches were implemented in this fork of pg_probackup. The second -approach requires [patch for PostgreSQL 9.6.2](https://gist.github.com/alubennikova/9daacf35790eca1a09b63a1bca86d836) or -[patch for PostgreSQL 10 (master)](https://gist.github.com/alubennikova/d24f61804525f0248fa71a1075158c21). - -Testing block level incremental backup --------------------------------------- - -You need to apply ptrack patch to [PostgreSQL 9.6.2](https://gist.github.com/alubennikova/9daacf35790eca1a09b63a1bca86d836) -or [PostgreSQL 10 (master)](https://gist.github.com/alubennikova/d24f61804525f0248fa71a1075158c21). -Or you can build and install [PGPRO9_5 or PGPRO9_6 branch of PostgreSQL](https://github.com/postgrespro/postgrespro). -Note that PGPRO branches currently contain old version of ptrack. - -### Retrieving changed blocks from WAL archive - -You need to enable WAL archive by adding following lines to postgresql.conf: - -``` -wal_level = archive -archive_mode = on -archive_command = 'test ! -f /home/postgres/backup/wal/%f && cp %p /home/postgres/backup/wal/%f' -``` - -Example backup (assuming PostgreSQL is running): -```bash -# Init pg_aramn backup folder -pg_probackup init -B /home/postgres/backup -# Make full backup with 2 thread and verbose mode. -pg_probackup backup -B /home/postgres/backup -D /home/postgres/pgdata -b full -v -j 2 -# Show backups information -pg_probackup show -B /home/postgres/backup - -# Now you can insert or update some data in your database - -# Then start the incremental backup. -pg_probackup backup -B /home/postgres/backup -D /home/postgres/pgdata -b page -v -j 2 -# You should see that increment is really small -pg_probackup show -B /home/postgres/backup -``` - -For restore after remove your pgdata you can use: -``` -pg_probackup restore -B /home/postgres/backup -D /home/postgres/pgdata -j 4 --verbose -``` - -### Retrieving changed blocks from ptrack - -The advantage of this approach is that you don't have to save WAL archive. You will need to enable ptrack in postgresql.conf (restart required). - -``` -ptrack_enable = on -``` - -Also, some WALs still need to be fetched in order to get consistent backup. pg_probackup can fetch them trough the streaming replication protocol. Thus, you also need to [enable streaming replication connection](https://wiki.postgresql.org/wiki/Streaming_Replication). - -Example backup (assuming PostgreSQL is running): -```bash -# Init pg_aramn backup folder -pg_probackup init -B /home/postgres/backup -# Make full backup with 2 thread and verbose mode. -pg_probackup backup -B /home/postgres/backup -D /home/postgres/pgdata -b full -v -j 2 --stream -# Show backups information -pg_probackup show -B /home/postgres/backup - -# Now you can insert or update some data in your database - -# Then start the incremental backup. -pg_probackup backup -B /home/postgres/backup -D /home/postgres/pgdata -b ptrack -v -j 2 --stream -# You should see that increment is really small -pg_probackup show -B /home/postgres/backup -``` - -For restore after remove your pgdata you can use: -``` -pg_probackup restore -B /home/postgres/backup -D /home/postgres/pgdata -j 4 --verbose --stream -``` - -License -------- - -pg_probackup can be distributed under the PostgreSQL license. See COPYRIGHT -file for more information. pg_arman is a fork of the existing project -pg_rman, initially created and maintained by NTT and Itagaki Takahiro. diff --git a/src/archive.c b/src/archive.c index 1bb17643..4b9cd18a 100644 --- a/src/archive.c +++ b/src/archive.c @@ -66,6 +66,9 @@ do_archive_push(char *wal_file_path, char *wal_file_name) join_path_components(backup_wal_file_path, arclog_path, wal_file_name); elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + if (access(backup_wal_file_path, F_OK) != -1) + elog(ERROR, "file '%s', already exists.", backup_wal_file_path); + copy_wal_file(absolute_wal_file_path, backup_wal_file_path); elog(INFO, "pg_probackup archive-push completed successfully"); diff --git a/src/backup.c b/src/backup.c index e8cb56e1..368f12e4 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1112,6 +1112,7 @@ pg_stop_backup(pgBackup *backup) backup_label, strerror(errno)); fwrite(PQgetvalue(res, 0, 1), 1, strlen(PQgetvalue(res, 0, 1)), fp); + fsync(fileno(fp)); fclose(fp); /* @@ -1139,6 +1140,7 @@ pg_stop_backup(pgBackup *backup) tablespace_map, strerror(errno)); fwrite(PQgetvalue(res, 0, 2), 1, strlen(PQgetvalue(res, 0, 2)), fp); + fsync(fileno(fp)); fclose(fp); file = pgFileNew(tablespace_map, true); @@ -1662,6 +1664,7 @@ write_backup_file_list(parray *files, const char *root) print_file_list(fp, files, root); + fsync(fileno(fp)); fclose(fp); } diff --git a/src/data.c b/src/data.c index bcf89d94..c6bcceee 100644 --- a/src/data.c +++ b/src/data.c @@ -357,6 +357,7 @@ backup_data_file(const char *from_root, const char *to_root, strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); @@ -470,6 +471,7 @@ restore_file_partly(const char *from_root,const char *to_root, pgFile *file) strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); } @@ -605,6 +607,7 @@ restore_data_file(const char *from_root, strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); } @@ -732,6 +735,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); @@ -832,9 +836,9 @@ copy_wal_file(const char *from_path, const char *to_path) strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); - } /* @@ -957,6 +961,7 @@ copy_file_partly(const char *from_root, const char *to_root, /* add meta information needed for recovery */ file->is_partial_copy = true; + fsync(fileno(out)); fclose(in); fclose(out); diff --git a/src/parsexlog.c b/src/parsexlog.c index aa8de7ed..6aadfaea 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -319,7 +319,7 @@ validate_wal(pgBackup *backup, * If recovery target is provided check that we can restore backup to a * recoverty target time or xid. */ - if (!TransactionIdIsValid(target_xid) || target_time == 0) + if (!TransactionIdIsValid(target_xid) && target_time == 0) { /* Recoverty target is not given so exit */ elog(INFO, "backup validation completed successfully"); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 867df105..7547f679 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -17,7 +17,7 @@ #include #include -const char *PROGRAM_VERSION = "1.1.17"; +const char *PROGRAM_VERSION = "2.0.0"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; @@ -72,6 +72,7 @@ uint32 retention_window = 0; /* compression options */ CompressAlg compress_alg = NOT_DEFINED_COMPRESS; int compress_level = DEFAULT_COMPRESS_LEVEL; +bool compress_shortcut = false; /* other options */ char *instance_name; @@ -132,6 +133,7 @@ static pgut_option options[] = /* compression options */ { 'f', 36, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, { 'u', 37, "compress-level", &compress_level, SOURCE_CMDLINE }, + { 'b', 38, "compress", &compress_shortcut, SOURCE_CMDLINE }, /* logging options */ { 'f', 40, "log-level", opt_log_level, SOURCE_CMDLINE }, { 's', 41, "log-filename", &log_filename, SOURCE_CMDLINE }, @@ -353,6 +355,9 @@ main(int argc, char *argv[]) if (num_threads < 1) num_threads = 1; + if (compress_shortcut) + compress_alg = ZLIB_COMPRESS; + if (backup_subcmd != SET_CONFIG) { if (compress_level != DEFAULT_COMPRESS_LEVEL @@ -405,7 +410,7 @@ main(int argc, char *argv[]) elog(ERROR, "show-config command doesn't accept any options except -B and --instance"); return do_configure(true); case SET_CONFIG: - if (argc == 5) + if (argc == 6) elog(ERROR, "set-config command requires at least one option"); return do_configure(false); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6e2b42ec..efcb928e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -294,6 +294,7 @@ extern uint32 retention_window; /* compression options */ extern CompressAlg compress_alg; extern int compress_level; +extern bool compress_shortcut; #define DEFAULT_COMPRESS_LEVEL 6 diff --git a/src/restore.c b/src/restore.c index cc012a9d..69a614dc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -750,6 +750,7 @@ create_recovery_conf(time_t backup_id, if (target_tli) fprintf(fp, "recovery_target_timeline = '%u'\n", target_tli); + fsync(fileno(fp)); fclose(fp); } diff --git a/src/validate.c b/src/validate.c index 0795179a..f9eeb3fa 100644 --- a/src/validate.c +++ b/src/validate.c @@ -230,7 +230,10 @@ do_validate_all(void) } if (corrupted_backup_found) + { elog(INFO, "Some backups are not valid"); + return 1; + } else elog(INFO, "All backups are valid"); diff --git a/tests/__init__.py b/tests/__init__.py index e6094ad0..db736768 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,16 +6,11 @@ from . import init_test, option_test, show_test, \ ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ - pgpro688, false_positive, replica + false_positive, replica def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - suite.addTests(loader.loadTestsFromModule(replica)) -# suite.addTests(loader.loadTestsFromModule(pgpro560)) -# suite.addTests(loader.loadTestsFromModule(pgpro589)) -# suite.addTests(loader.loadTestsFromModule(pgpro688)) -# suite.addTests(loader.loadTestsFromModule(false_positive)) suite.addTests(loader.loadTestsFromModule(init_test)) suite.addTests(loader.loadTestsFromModule(option_test)) suite.addTests(loader.loadTestsFromModule(show_test)) @@ -33,8 +28,9 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility)) suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full)) suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) + suite.addTests(loader.loadTestsFromModule(replica)) + suite.addTests(loader.loadTestsFromModule(pgpro560)) + suite.addTests(loader.loadTestsFromModule(pgpro589)) + suite.addTests(loader.loadTestsFromModule(false_positive)) return suite - - -# ExpectedFailures are bugs, which should be fixed diff --git a/tests/backup_test.py b/tests/backup_test.py index b68be8d2..da31f25f 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -1,9 +1,7 @@ import unittest import os -import six from time import sleep -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import stop_all +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException class BackupTest(ProbackupTest, unittest.TestCase): @@ -12,10 +10,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): super(BackupTest, self).__init__(*args, **kwargs) self.module_name = 'backup' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure # PGPRO-707 @@ -39,8 +33,8 @@ class BackupTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] - self.assertEqual(show_backup['Status'], six.b("OK")) - self.assertEqual(show_backup['Mode'], six.b("FULL")) + self.assertEqual(show_backup['Status'], "OK") + self.assertEqual(show_backup['Mode'], "FULL") # postmaster.pid and postmaster.opts shouldn't be copied excluded = True @@ -56,8 +50,8 @@ class BackupTest(ProbackupTest, unittest.TestCase): # print self.show_pb(node) show_backup = self.show_pb(backup_dir, 'node')[1] - self.assertEqual(show_backup['Status'], six.b("OK")) - self.assertEqual(show_backup['Mode'], six.b("PAGE")) + self.assertEqual(show_backup['Status'], "OK") + self.assertEqual(show_backup['Mode'], "PAGE") # Check parent backup self.assertEqual( @@ -68,15 +62,16 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node, backup_type="ptrack") show_backup = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup['Status'], six.b("OK")) - self.assertEqual(show_backup['Mode'], six.b("PTRACK")) + self.assertEqual(show_backup['Status'], "OK") + self.assertEqual(show_backup['Mode'], "PTRACK") # Check parent backup self.assertEqual( page_backup_id, self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_smooth_checkpoint(self): @@ -93,9 +88,12 @@ class BackupTest(ProbackupTest, unittest.TestCase): node.start() self.backup_node(backup_dir, 'node' ,node, options=["-C"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) + #@unittest.skip("skip") def test_incremental_backup_without_full(self): """page-level backup without validated full backup""" @@ -115,7 +113,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -127,16 +125,17 @@ class BackupTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR")) - node.stop() + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") - @unittest.expectedFailure - # Need to forcibly validate parent + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + # @unittest.expectedFailure def test_incremental_backup_corrupt_full(self): """page-level backup with corrupted full backup""" fname = self.id().split('.')[3] @@ -151,29 +150,37 @@ class BackupTest(ProbackupTest, unittest.TestCase): node.start() backup_id = self.backup_node(backup_dir, 'node', node) - file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + file = os.path.join(backup_dir, "backups", "node", backup_id, "database", "postgresql.conf") os.remove(file) + try: + self.validate_pb(backup_dir, 'node') + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of validation of corrupted backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue("INFO: Validate backups of the instance 'node'\n" in e.message + and 'WARNING: Backup file "{0}" is not found\n'.format(file) in e.message + and "WARNING: Backup {0} is corrupted\n".format(backup_id) in e.message + and "INFO: Some backups are not valid\n" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd)) + try: self.backup_node(backup_dir, 'node', node, backup_type="page") # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + "ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n", + "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd)) - sleep(1) - self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException, e: - self.assertEqual(e.message, - 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # sleep(1) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "ERROR") - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR")) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_ptrack_threads(self): @@ -190,12 +197,13 @@ class BackupTest(ProbackupTest, unittest.TestCase): node.start() self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_ptrack_threads_stream(self): @@ -213,7 +221,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK")) - node.stop() + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/delete_test.py b/tests/delete_test.py index e9c176f0..593bd83c 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -1,8 +1,6 @@ import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import stop_all +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess @@ -12,10 +10,6 @@ class DeleteTest(ProbackupTest, unittest.TestCase): super(DeleteTest, self).__init__(*args, **kwargs) self.module_name = 'delete' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_delete_full_backups(self): @@ -55,7 +49,8 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.assertEqual(show_backups[0]['ID'], id_1) self.assertEqual(show_backups[1]['ID'], id_3) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_delete_increment_page(self): """delete increment and all after him""" @@ -88,12 +83,13 @@ class DeleteTest(ProbackupTest, unittest.TestCase): show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], six.b("FULL")) - self.assertEqual(show_backups[0]['Status'], six.b("OK")) - self.assertEqual(show_backups[1]['Mode'], six.b("FULL")) - self.assertEqual(show_backups[1]['Status'], six.b("OK")) + self.assertEqual(show_backups[0]['Mode'], "FULL") + self.assertEqual(show_backups[0]['Status'], "OK") + self.assertEqual(show_backups[1]['Mode'], "FULL") + self.assertEqual(show_backups[1]['Status'], "OK") - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_delete_increment_ptrack(self): """delete increment and all after him""" @@ -126,9 +122,10 @@ class DeleteTest(ProbackupTest, unittest.TestCase): show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], six.b("FULL")) - self.assertEqual(show_backups[0]['Status'], six.b("OK")) - self.assertEqual(show_backups[1]['Mode'], six.b("FULL")) - self.assertEqual(show_backups[1]['Status'], six.b("OK")) + self.assertEqual(show_backups[0]['Mode'], "FULL") + self.assertEqual(show_backups[0]['Status'], "OK") + self.assertEqual(show_backups[1]['Mode'], "FULL") + self.assertEqual(show_backups[1]['Status'], "OK") - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index e88cb7c9..4f95ebcf 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 1.1.17 +pg_probackup 2.0.0 diff --git a/tests/false_positive.py b/tests/false_positive.py index 71e2899f..00477524 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -1,11 +1,8 @@ import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit class FalsePositive(ProbackupTest, unittest.TestCase): @@ -14,10 +11,6 @@ class FalsePositive(ProbackupTest, unittest.TestCase): super(FalsePositive, self).__init__(*args, **kwargs) self.module_name = 'false_positive' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_pgpro561(self): @@ -26,62 +19,64 @@ class FalsePositive(ProbackupTest, unittest.TestCase): check that archiving is not successful on node1 """ fname = self.id().split('.')[3] - master = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/master".format(fname), - set_archiving=True, + node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} ) - master.start() + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + self.set_archiving(backup_dir, 'node1', node1) + node1.start() - self.assertEqual(self.init_pb(master), six.b("")) - id = self.backup_pb(master, backup_type='full', options=["--stream"]) + backup_id = self.backup_node(backup_dir, 'node1', node1, options=["--stream"]) - node1 = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/node1".format(fname)) - node1.cleanup() + node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(self.module_name, fname)) + node2.cleanup() - master.psql( + node1.psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") - self.backup_pb(master, backup_type='page', options=["--stream"]) - self.restore_pb(backup_dir=self.backup_dir(master), data_dir=node1.data_dir) - node1.append_conf('postgresql.auto.conf', 'port = {0}'.format(node1.port)) - node1.start({"-t": "600"}) + self.backup_node(backup_dir, 'node1', node1, backup_type='page', options=["--stream"]) + self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + node2.append_conf('postgresql.auto.conf', 'port = {0}'.format(node2.port)) + node2.start({"-t": "600"}) - timeline_master = master.get_control_data()["Latest checkpoint's TimeLineID"] timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] - self.assertEqual(timeline_master, timeline_node1, "Timelines on Master and Node1 should be equal. This is unexpected") + timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual(timeline_node1, timeline_node2, "Timelines on Master and Node1 should be equal. This is unexpected") - archive_command_master = master.safe_psql("postgres", "show archive_command") archive_command_node1 = node1.safe_psql("postgres", "show archive_command") - self.assertEqual(archive_command_master, archive_command_node1, "Archive command on Master and Node should be equal. This is unexpected") + archive_command_node2 = node2.safe_psql("postgres", "show archive_command") + self.assertEqual(archive_command_node1, archive_command_node2, "Archive command on Master and Node should be equal. This is unexpected") - res = node1.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") # self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) - if res == six.b(""): + if result == "": self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') - master.stop() - node1.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) + # @unittest.skip("skip") def pgpro688(self): - """ - make node with archiving, make backup, - get Recovery Time, validate to Recovery Time - Waiting PGPRO-688 - """ + """make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - id = self.backup_pb(node, backup_type='full') - recovery_time = self.show_pb(node, id=id)['recovery-time'] + backup_id = self.backup_node(backup_dir, 'node', node) + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] # Uncommenting this section will make this test True Positive #node.psql("postgres", "select pg_create_restore_point('123')") @@ -89,46 +84,58 @@ class FalsePositive(ProbackupTest, unittest.TestCase): #node.psql("postgres", "select pg_switch_xlog()") #### - try: - self.validate_pb(node, options=["--time='{0}'".format(recovery_time)]) - self.assertEqual(1, 0, 'Error is expected because We should not be able safely validate "Recovery Time" without wal record with timestamp') - except ProbackupException, e: - self.assertTrue('WARNING: recovery can be done up to time {0}'.format(recovery_time) in e.message) + #try: + self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(recovery_time)]) + # we should die here because exception is what we expect to happen + # self.assertEqual(1, 0, "Expecting Error because it should not be possible safely validate 'Recovery Time' without wal record with timestamp.\n Output: {0} \n CMD: {1}".format( + # repr(self.output), self.cmd)) + # except ProbackupException as e: + # self.assertTrue('WARNING: recovery can be done up to time {0}'.format(recovery_time) in e.message, + # '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) + # @unittest.skip("skip") def pgpro702_688(self): - """ - make node without archiving, make stream backup, - get Recovery Time, validate to Recovery Time - """ + """make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - id = self.backup_pb(node, backup_type='full', options=["--stream"]) - recovery_time = self.show_pb(node, id=id)['recovery-time'] + backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, options=["--time='{0}'".format(recovery_time)])) + self.validate_pb(backup_dir, 'node', node, options=["--time='{0}'".format(recovery_time)])) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure def test_validate_wal_lost_segment(self): """Loose segment located between backups. ExpectedFailure. This is BUG """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - self.backup_pb(node, backup_type='full') + + self.backup_node(backup_dir, 'node', node) # make some wals node.pgbench_init(scale=2) @@ -141,15 +148,59 @@ class FalsePositive(ProbackupTest, unittest.TestCase): pgbench.stdout.close() # delete last wal segment - wals_dir = os.path.join(self.backup_dir(node), "wal") + wals_dir = os.path.join(backup_dir, "wal", 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals = map(int, wals) - os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))) + os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) ##### Hole Smokes, Batman! We just lost a wal segment and know nothing about it ##### We need archive-push ASAP - self.backup_pb(node, backup_type='full') - self.assertTrue('validation completed successfully' in self.validate_pb(node)) + self.backup_node(backup_dir, 'node', node) + self.assertFalse('validation completed successfully' in self.validate_pb(backup_dir, 'node')) ######## - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + @unittest.expectedFailure + # Need to force validation of ancestor-chain + def test_incremental_backup_corrupt_full_1(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + sleep(1) + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b7a7cf3b..d27b3181 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1,10 +1,10 @@ # you need os for unittest to work import os -from sys import exit +from sys import exit, argv, version_info import subprocess import shutil import six -from testgres import get_new_node +from testgres import get_new_node, clean_all import hashlib import re import pwd @@ -73,40 +73,6 @@ def dir_files(base_dir): out_list.sort() return out_list - -class ShowBackup(object): - def __init__(self, line): - self.counter = 0 - - print split_line - self.id = self.get_inc(split_line) - # TODO: parse to datetime - if len(split_line) == 12: - self.recovery_time = "%s %s" % (self.get_inc(split_line), self.get_inc(split_line)) - # if recovery time is '----' - else: - self.recovery_time = self.get_inc(split_line) - self.mode = self.get_inc(split_line) -# print self.mode - self.wal = self.get_inc(split_line) - self.cur_tli = self.get_inc(split_line) - # slash - self.counter += 1 - self.parent_tli = self.get_inc(split_line) - # TODO: parse to interval - self.time = self.get_inc(split_line) - # TODO: maybe rename to size? - self.data = self.get_inc(split_line) - self.start_lsn = self.get_inc(split_line) - self.stop_lsn = self.get_inc(split_line) - self.status = self.get_inc(split_line) - - def get_inc(self, split_line): -# self.counter += 1 -# return split_line[self.counter - 1] - return split_line - - class ProbackupTest(object): def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) @@ -134,7 +100,6 @@ class ProbackupTest(object): self.test_env["LC_MESSAGES"] = "C" self.test_env["LC_TIME"] = "C" - self.helpers_path = os.path.dirname(os.path.realpath(__file__)) self.dir_path = os.path.abspath(os.path.join(self.helpers_path, os.pardir)) self.tmp_path = os.path.abspath(os.path.join(self.dir_path, 'tmp_dirs')) @@ -145,6 +110,10 @@ class ProbackupTest(object): self.probackup_path = os.path.abspath(os.path.join( self.dir_path, "../pg_probackup")) self.user = self.get_username() + if '-v' in argv or '--verbose' in argv: + self.verbose = True + else: + self.verbose = False def arcwal_dir(self, node): return "%s/backup/wal" % node.base_dir @@ -228,10 +197,12 @@ class ProbackupTest(object): byte_size_minus_header = byte_size - header_size file = os.open(file + '_ptrack', os.O_RDONLY) os.lseek(file, header_size, 0) - lot_of_bytes = os.read(file, byte_size_minus_header) - for byte in lot_of_bytes: + lots_of_bytes = os.read(file, byte_size_minus_header) + byte_list = [lots_of_bytes[i:i+1] for i in range(len(lots_of_bytes))] + for byte in byte_list: + #byte_inverted = bin(int(byte, base=16))[2:][::-1] + #bits = (byte >> x) & 1 for x in range(7, -1, -1) byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1] -# byte_to_bits = (byte >> x) & 1 for x in range(7, -1, -1) for bit in byte_inverted: if len(ptrack_bits_for_fork) < size: ptrack_bits_for_fork.append(int(bit)) @@ -249,9 +220,10 @@ class ProbackupTest(object): # Page was not present before, meaning that relation got bigger # Ptrack should be equal to 1 if idx_dict['ptrack'][PageNum] != 1: - print 'Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict + if self.verbose: + print('Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) success = False continue if PageNum not in idx_dict['new_pages']: @@ -266,29 +238,34 @@ class ProbackupTest(object): if idx_dict['new_pages'][PageNum] != idx_dict['old_pages'][PageNum]: # Page has been changed, meaning that ptrack should be equal to 1 if idx_dict['ptrack'][PageNum] != 1: - print 'Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict + if self.verbose: + print('Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) if PageNum == 0 and idx_dict['type'] == 'spgist': - print 'SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0' + if self.verbose: + print('SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0') continue success = False else: # Page has not been changed, meaning that ptrack should be equal to 0 if idx_dict['ptrack'][PageNum] != 0: - print 'Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict - self.assertEqual(success, True) + if self.verbose: + print('Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) + self.assertEqual(success, True, 'Ptrack of index {0} does not correspond to state of its pages.\n Gory Details: \n{1}'.format( + idx_dict['type'], idx_dict)) def check_ptrack_recovery(self, idx_dict): success = True size = idx_dict['size'] for PageNum in range(size): if idx_dict['ptrack'][PageNum] != 1: - print 'Recovery for Page Number {0} of Type {1} was conducted, but ptrack value is {2}. THIS IS BAD'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict + if self.verbose: + print('Recovery for Page Number {0} of Type {1} was conducted, but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) success = False self.assertEqual(success, True) @@ -296,29 +273,23 @@ class ProbackupTest(object): success = True for PageNum in range(size): if idx_dict['ptrack'][PageNum] != 0: - print 'Ptrack for Page Number {0} of Type {1} should be clean, but ptrack value is {2}. THIS IS BAD'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict + if self.verbose: + print('Ptrack for Page Number {0} of Type {1} should be clean, but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) success = False - self.assertEqual(success, True) + self.assertEqual(success, True, '') - def run_pb(self, command, async=False): + def run_pb(self, command): try: self.cmd = [' '.join(map(str,[self.probackup_path] + command))] - print self.cmd - if async is True: - return subprocess.Popen( - [self.probackup_path] + command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.test_env - ) - else: - self.output = subprocess.check_output( - [self.probackup_path] + command, - stderr=subprocess.STDOUT, - env=self.test_env - ) + if self.verbose: + print(self.cmd) + self.output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") if command[0] == 'backup': # return backup ID for line in self.output.splitlines(): @@ -327,7 +298,7 @@ class ProbackupTest(object): else: return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output, self.cmd) + raise ProbackupException(e.output.decode("utf-8"), self.cmd) def init_pb(self, backup_dir): @@ -358,12 +329,21 @@ class ProbackupTest(object): def clean_pb(self, backup_dir): shutil.rmtree(backup_dir, ignore_errors=True) - def backup_node(self, backup_dir, instance, node, backup_type="full", options=[], async=False): + def backup_node(self, backup_dir, instance, node=False, data_dir=False, backup_type="full", options=[]): + if not node and not data_dir: + print('You must provide ether node or data_dir for backup') + exit(1) + + if node: + pgdata = node.data_dir + + if data_dir: + pgdata = data_dir cmd_list = [ "backup", "-B", backup_dir, - "-D", node.data_dir, + "-D", pgdata, "-p", "%i" % node.port, "-d", "postgres", "--instance={0}".format(instance) @@ -371,7 +351,7 @@ class ProbackupTest(object): if backup_type: cmd_list += ["-b", backup_type] - return self.run_pb(cmd_list + options, async) + return self.run_pb(cmd_list + options) def restore_node(self, backup_dir, instance, node=False, data_dir=None, backup_id=None, options=[]): if data_dir is None: @@ -429,9 +409,9 @@ class ProbackupTest(object): if i == '': backup_record_split.remove(i) if len(header_split) != len(backup_record_split): - print warning.format( + print(warning.format( header=header, body=body, - header_split=header_split, body_split=backup_record_split) + header_split=header_split, body_split=backup_record_split)) exit(1) new_dict = dict(zip(header_split, backup_record_split)) backup_list.append(new_dict) @@ -473,7 +453,6 @@ class ProbackupTest(object): if backup_id: cmd_list += ["-i", backup_id] - # print(cmd_list) return self.run_pb(cmd_list + options) def delete_expired(self, backup_dir, instance, options=[]): @@ -563,3 +542,17 @@ class ProbackupTest(object): def get_username(self): """ Returns current user name """ return pwd.getpwuid(os.getuid())[0] + + def del_test_dir(self, module_name, fname): + """ Returns current user name """ + try: + clean_all() + except: + pass + + shutil.rmtree(os.path.join(self.tmp_path, self.module_name, fname), + ignore_errors=True) + try: + os.rmdir(os.path.join(self.tmp_path, self.module_name)) + except: + pass diff --git a/tests/init_test.py b/tests/init_test.py index b19f069d..52699f4f 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -1,9 +1,6 @@ -import unittest -from sys import exit import os -from os import path -import six -from helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException +import unittest +from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException class InitTest(ProbackupTest, unittest.TestCase): @@ -25,20 +22,22 @@ class InitTest(ProbackupTest, unittest.TestCase): ['backups', 'wal'] ) self.add_instance(backup_dir, 'node', node) - - self.assertEqual("INFO: Instance 'node' successfully deleted\n", - self.del_instance(backup_dir, 'node', node), + self.assertEqual("INFO: Instance 'node' successfully deleted\n", self.del_instance(backup_dir, 'node', node), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) try: self.show_pb(backup_dir, 'node') self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, "ERROR: Instance 'node' does not exist in this backup catalog\n", '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + # @unittest.skip("skip") def test_already_exist(self): """Failure with backup catalog already existed""" fname = self.id().split(".")[3] @@ -49,25 +48,28 @@ class InitTest(ProbackupTest, unittest.TestCase): self.show_pb(backup_dir, 'node') self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, "ERROR: Instance 'node' does not exist in this backup catalog\n", '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + # @unittest.skip("skip") def test_abs_path(self): """failure with backup catalog should be given as absolute path""" fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname)) try: - self.run_pb(["init", "-B", path.relpath("%s/backup" % node.base_dir, self.dir_path)]) + self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, "ERROR: -B, --backup-path must be an absolute path\n", '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/option_test.py b/tests/option_test.py index 1114c169..a05a0c44 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -1,8 +1,6 @@ import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import stop_all +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException class OptionTest(ProbackupTest, unittest.TestCase): @@ -11,10 +9,6 @@ class OptionTest(ProbackupTest, unittest.TestCase): super(OptionTest, self).__init__(*args, **kwargs) self.module_name = 'option' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_help_1(self): @@ -24,7 +18,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: self.assertEqual( self.run_pb(["--help"]), - help_out.read() + help_out.read().decode("utf-8") ) # @unittest.skip("skip") @@ -35,7 +29,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: self.assertEqual( self.run_pb(["--version"]), - version_out.read() + version_out.read().decode("utf-8") ) # @unittest.skip("skip") @@ -47,7 +41,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.run_pb(["backup", "-b", "full"]) self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -59,7 +53,6 @@ class OptionTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}) - try: node.stop() except: @@ -73,7 +66,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: required parameter not specified: --instance\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -83,7 +76,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -93,23 +86,25 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: invalid backup-mode "bad"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # delete failure without ID try: self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: required backup ID not specified\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + #@unittest.skip("skip") def test_options_5(self): """check options test""" @@ -118,7 +113,8 @@ class OptionTest(ProbackupTest, unittest.TestCase): node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}) - self.assertEqual(self.init_pb(backup_dir), six.b("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir))) + self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir), + self.init_pb(backup_dir)) self.add_instance(backup_dir, 'node', node) node.start() @@ -131,7 +127,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: syntax error in " = INFINITE"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -149,7 +145,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: invalid backup-mode ""\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -162,10 +158,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy=1\n") - self.assertEqual( - self.show_config(backup_dir, 'node')['retention-redundancy'], - six.b('1') - ) + self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') # User cannot send --system-identifier parameter via command line try: @@ -173,7 +166,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: option system-identifier cannot be specified in command line\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -187,7 +180,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -205,10 +198,10 @@ class OptionTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: invalid option "TIMELINEID"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) -# self.clean_pb(backup_dir) -# node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/pb_lib.py b/tests/pb_lib.py deleted file mode 100644 index 9406e9b6..00000000 --- a/tests/pb_lib.py +++ /dev/null @@ -1,304 +0,0 @@ -import os -from os import path, listdir -import subprocess -import shutil -import six -from testgres import get_new_node - - -def dir_files(base_dir): - out_list = [] - for dir_name, subdir_list, file_list in os.walk(base_dir): - if dir_name != base_dir: - out_list.append(path.relpath(dir_name, base_dir)) - for fname in file_list: - out_list.append(path.relpath(path.join(dir_name, fname), base_dir)) - out_list.sort() - return out_list - - -class ShowBackup(object): - def __init__(self, split_line): - self.counter = 0 - - self.id = self.get_inc(split_line) - # TODO: parse to datetime - if len(split_line) == 12: - self.recovery_time = "%s %s" % (self.get_inc(split_line), - self.get_inc(split_line)) - # if recovery time is '----' - else: - self.recovery_time = self.get_inc(split_line) - self.mode = self.get_inc(split_line) - self.cur_tli = self.get_inc(split_line) - # slash - self.counter += 1 - self.parent_tli = self.get_inc(split_line) - # TODO: parse to interval - self.time = self.get_inc(split_line) - # TODO: maybe rename to size? - self.data = self.get_inc(split_line) - self.start_lsn = self.get_inc(split_line) - self.stop_lsn = self.get_inc(split_line) - self.status = self.get_inc(split_line) - - def get_inc(self, split_line): - self.counter += 1 - return split_line[self.counter - 1] - - -class ProbackupTest(object): - def __init__(self, *args, **kwargs): - super(ProbackupTest, self).__init__(*args, **kwargs) - self.test_env = os.environ.copy() - envs_list = [ - "LANGUAGE", - "LC_ALL", - "PGCONNECT_TIMEOUT", - "PGDATA", - "PGDATABASE", - "PGHOSTADDR", - "PGREQUIRESSL", - "PGSERVICE", - "PGSSLMODE", - "PGUSER", - "PGPORT", - "PGHOST" - ] - - for e in envs_list: - try: - del self.test_env[e] - except: - pass - - self.test_env["LC_MESSAGES"] = "C" - self.test_env["LC_TIME"] = "C" - - self.dir_path = path.dirname(os.path.realpath(__file__)) - try: - os.makedirs(path.join(self.dir_path, "tmp_dirs")) - except: - pass - self.probackup_path = os.path.abspath(path.join( - self.dir_path, - "../pg_probackup" - )) - - def arcwal_dir(self, node): - return "%s/backup/wal" % node.base_dir - - def backup_dir(self, node): - return os.path.abspath("%s/backup" % node.base_dir) - - def make_bnode(self, base_dir=None, allows_streaming=False, options={}): - real_base_dir = path.join(self.dir_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - - node = get_new_node('test', base_dir=real_base_dir) - node.init(allows_streaming=allows_streaming) - - if not allows_streaming: - node.append_conf("postgresql.conf", "wal_level = hot_standby") - node.append_conf("postgresql.conf", "archive_mode = on") - node.append_conf( - "postgresql.conf", - """archive_command = 'cp "%%p" "%s/%%f"'""" % os.path.abspath(self.arcwal_dir(node)) - ) - - for key, value in six.iteritems(options): - node.append_conf("postgresql.conf", "%s = %s" % (key, value)) - - return node - - def make_bnode_replica(self, root_node, base_dir=None, options={}): - real_base_dir = path.join(self.dir_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - - root_node.backup("basebackup") - - replica = get_new_node("replica", base_dir=real_base_dir) - # replica.init_from_backup(root_node, "data_replica", has_streaming=True) - - # Move data from backup - backup_path = os.path.join(root_node.base_dir, "basebackup") - shutil.move(backup_path, replica.data_dir) - os.chmod(replica.data_dir, 0o0700) - - # Change port in config file - replica.append_conf( - "postgresql.conf", - "port = {}".format(replica.port) - ) - # Enable streaming - replica.enable_streaming(root_node) - - for key, value in six.iteritems(options): - replica.append_conf("postgresql.conf", "%s = %s" % (key, value)) - - return replica - - def run_pb(self, command): - try: - return subprocess.check_output( - [self.probackup_path] + command, - stderr=subprocess.STDOUT, - env=self.test_env - ) - except subprocess.CalledProcessError as err: - return err.output - - def init_pb(self, node): - return self.run_pb([ - "init", - "-B", self.backup_dir(node), - "-D", node.data_dir - ]) - - def clean_pb(self, node): - shutil.rmtree(self.backup_dir(node), ignore_errors=True) - - def backup_pb(self, node, backup_type="full", options=[]): - cmd_list = [ - "backup", - "-D", node.data_dir, - "-B", self.backup_dir(node), - "-p", "%i" % node.port, - "-d", "postgres" - ] - if backup_type: - cmd_list += ["-b", backup_type] - - return self.run_pb(cmd_list + options) - - def backup_pb_proc(self, node, backup_dir, backup_type="full", - stdout=None, stderr=None, options=[]): - cmd_list = [ - self.probackup_path, - "backup", - "-D", node.data_dir, - "-B", backup_dir, - "-p", "%i" % (node.port), - "-d", "postgres" - ] - if backup_type: - cmd_list += ["-b", backup_type] - - proc = subprocess.Popen( - cmd_list + options, - stdout=stdout, - stderr=stderr - ) - - return proc - - def restore_pb(self, node, id=None, options=[]): - cmd_list = [ - "-D", node.data_dir, - "-B", self.backup_dir(node), - "restore" - ] - if id: - cmd_list.append(id) - - # print(cmd_list) - return self.run_pb(cmd_list + options) - - def show_pb(self, node, id=None, options=[], as_text=False): - cmd_list = [ - "show", - "-B", self.backup_dir(node), - ] - if id: - cmd_list += [id] - - # print(cmd_list) - if as_text: - return self.run_pb(options + cmd_list) - elif id is None: - return [ShowBackup(line.split()) for line in self.run_pb(options + cmd_list).splitlines()[3:]] - else: - return dict([ - line.split(six.b("=")) - for line in self.run_pb(options + cmd_list).splitlines() - if line[0] != six.b("#")[0] - ]) - - def validate_pb(self, node, id=None, options=[]): - cmd_list = [ - "-B", self.backup_dir(node), - "validate", - ] - if id: - cmd_list += [id] - - # print(cmd_list) - return self.run_pb(options + cmd_list) - - def delete_pb(self, node, id=None, options=[]): - cmd_list = [ - "-B", self.backup_dir(node), - "delete", - ] - if id: - cmd_list += [id] - - # print(cmd_list) - return self.run_pb(options + cmd_list) - - def retention_purge_pb(self, node, options=[]): - cmd_list = [ - "-B", self.backup_dir(node), - "retention", "purge", - ] - - return self.run_pb(options + cmd_list) - - def retention_show(self, node, options=[]): - cmd_list = [ - "-B", self.backup_dir(node), - "retention", "show", - ] - - return self.run_pb(options + cmd_list) - - def get_control_data(self, node): - pg_controldata = node.get_bin_path("pg_controldata") - out_data = {} - lines = subprocess.check_output( - [pg_controldata] + ["-D", node.data_dir], - stderr=subprocess.STDOUT, - env=self.test_env - ).splitlines() - for l in lines: - key, value = l.split(b":", maxsplit=1) - out_data[key.strip()] = value.strip() - return out_data - - def get_recovery_conf(self, node): - out_dict = {} - with open(path.join(node.data_dir, "recovery.conf"), "r") as recovery_conf: - for line in recovery_conf: - try: - key, value = line.split("=") - except: - continue - out_dict[key.strip()] = value.strip(" '").replace("'\n", "") - - return out_dict - - def wrong_wal_clean(self, node, wal_size): - wals_dir = path.join(self.backup_dir(node), "wal") - wals = [f for f in listdir(wals_dir) if path.isfile(path.join(wals_dir, f))] - wals.sort() - file_path = path.join(wals_dir, wals[-1]) - if path.getsize(file_path) != wal_size: - os.remove(file_path) - - def guc_wal_segment_size(self, node): - var = node.execute("postgres", "select setting from pg_settings where name = 'wal_segment_size'") - return int(var[0][0]) * self.guc_wal_block_size(node) - - def guc_wal_block_size(self, node): - var = node.execute("postgres", "select setting from pg_settings where name = 'wal_block_size'") - return int(var[0][0]) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index b0117f70..1654636e 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -1,21 +1,15 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit class CheckSystemID(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(CheckSystemID, self).__init__(*args, **kwargs) - - @classmethod - def tearDownClass(cls): - stop_all() + self.module_name = 'pgpro560' # @unittest.skip("skip") # @unittest.expectedFailure @@ -27,24 +21,32 @@ class CheckSystemID(ProbackupTest, unittest.TestCase): check that backup failed """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) file = os.path.join(node.base_dir,'data', 'global', 'pg_control') os.remove(file) try: - self.backup_pb(node, backup_type='full', options=['--stream']) - assertEqual(1, 0, 'Error is expected because of control file loss') - except ProbackupException, e: + self.backup_node(backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: self.assertTrue( - 'ERROR: could not open file' and 'pg_control' in e.message, - 'Expected error is about control file loss') + 'ERROR: could not open file' in e.message + and 'pg_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_pgpro560_systemid_mismatch(self): """ @@ -54,25 +56,33 @@ class CheckSystemID(ProbackupTest, unittest.TestCase): check that backup failed """ fname = self.id().split('.')[3] - node1 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node1".format(fname), + node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) node1.start() - node2 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node2".format(fname), + node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) node2.start() - self.assertEqual(self.init_pb(node1), six.b("")) + + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) try: - self.backup_pb(node1, data_dir=node2.data_dir, backup_type='full', options=['--stream']) - assertEqual(1, 0, 'Error is expected because of SYSTEM ID mismatch') - except ProbackupException, e: + self.backup_node(backup_dir, 'node1', node1, data_dir=node2.data_dir, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: self.assertTrue( - 'ERROR: Backup data directory was initialized for system id' and - 'but target backup directory system id is' in e.message, - 'Expected error is about SYSTEM ID mismatch') + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but target backup directory system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589.py index 00988d05..c15cd4aa 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -1,21 +1,15 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit class ArchiveCheck(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(ArchiveCheck, self).__init__(*args, **kwargs) - - @classmethod - def tearDownClass(cls): - stop_all() + self.module_name = 'pgpro589' # @unittest.skip("skip") # @unittest.expectedFailure @@ -26,10 +20,13 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): check ERROR text """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() node.pgbench_init(scale=5) @@ -41,15 +38,17 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): pgbench.wait() pgbench.stdout.close() - path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() - - self.assertEqual(self.init_pb(node), six.b("")) - try: - self.backup_pb(node, backup_type='full', options=['--archive-timeout=10']) - assertEqual(1, 0, 'Error is expected because of disabled archive_mode') - except ProbackupException, e: - self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n') + self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of disabled archive_mode.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_pgpro589(self): """ @@ -59,12 +58,16 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): check that no files where copied to backup catalogue """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - node.append_conf("postgresql.auto.conf", "archive_mode = on") - node.append_conf("postgresql.auto.conf", "wal_level = archive") + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + # make erroneus archive_command node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") node.start() @@ -76,22 +79,25 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): ) pgbench.wait() pgbench.stdout.close() - - path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() - self.assertEqual(self.init_pb(node), six.b("")) + path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip().decode("utf-8") try: - self.backup_pb( - node, backup_type='full', options=['--archive-timeout=10']) - assertEqual(1, 0, 'Error is expected because of missing archive wal segment with start_backup() LSN') - except ProbackupException, e: - self.assertTrue('INFO: wait for LSN' in e.message, "Expecting 'INFO: wait for LSN'") - self.assertTrue('ERROR: switched WAL segment' and 'could not be archived' in e.message, - "Expecting 'ERROR: switched WAL segment could not be archived'") + self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of missing archive wal segment with start_lsn.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: wait for LSN' in e.message + and 'ERROR: switched WAL segment' in e.message + and 'could not be archived' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - id = self.show_pb(node)[0]['ID'] - self.assertEqual('ERROR', self.show_pb(node, id=id)['status'], 'Backup should have ERROR status') - #print self.backup_dir(node) - file = os.path.join(self.backup_dir(node), 'backups', id, 'database', path) + backup_id = self.show_pb(backup_dir, 'node')[0]['ID'] + self.assertEqual('ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup should have ERROR status') + file = os.path.join(backup_dir, 'backups', 'node', backup_id, 'database', path) self.assertFalse(os.path.isfile(file), '\n Start LSN was not found in archive but datafiles where copied to backup catalogue.\n For example: {0}\n It is not optimal'.format(file)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/pgpro688.py b/tests/pgpro688.py deleted file mode 100644 index 416d2cf5..00000000 --- a/tests/pgpro688.py +++ /dev/null @@ -1,201 +0,0 @@ -import unittest -import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -from testgres import stop_all, get_username -import subprocess -from sys import exit, _getframe -import shutil -import time - - -class ReplicaTest(ProbackupTest, unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(ReplicaTest, self).__init__(*args, **kwargs) - self.module_name = 'replica' - self.instance_master = 'master' - self.instance_replica = 'replica' - -# @classmethod -# def tearDownClass(cls): -# stop_all() - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_replica_stream_full_backup(self): - """make full stream backup from replica""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') - master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'} - ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, self.instance_master, master) - master.start() - - # Make empty Object 'replica' from new node - replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname)) - replica_port = replica.port - replica.cleanup() - - # FULL STREAM backup of master - self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream']) - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") - before = master.execute("postgres", "SELECT * FROM t_heap") - - # FULL STREAM backup of master - self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream']) - - # Restore last backup from master to Replica directory - self.restore_node(backup_dir, self.instance_master, replica.data_dir) - # Set Replica - replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - replica.append_conf('recovery.conf', "standby_mode = 'on'") - replica.append_conf('recovery.conf', - "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port)) - replica.start({"-t": "600"}) - - # Check replica - after = replica.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Add instance replica - self.add_instance(backup_dir, self.instance_replica, replica) - - # FULL STREAM backup of replica - self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in - self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[ - '--stream', '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) - - # Validate instance replica - self.validate_pb(backup_dir, self.instance_replica) - self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) - - def test_replica_archive_full_backup(self): - """make page archive backup from replica""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') - master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'} - ) - self.set_archiving(backup_dir, self.instance_master, master) - self.init_pb(backup_dir) - self.add_instance(backup_dir, self.instance_master, master) - master.start() - - # Make empty Object 'replica' from new node - replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname)) - replica_port = replica.port - replica.cleanup() - - # FULL ARCHIVE backup of master - self.backup_node(backup_dir, self.instance_master, master, backup_type='full') - # Create table t_heap - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") - before = master.execute("postgres", "SELECT * FROM t_heap") - - # PAGE ARCHIVE backup of master - self.backup_node(backup_dir, self.instance_master, master, backup_type='page') - - # Restore last backup from master to Replica directory - self.restore_node(backup_dir, self.instance_master, replica.data_dir) - - # Set Replica - self.set_archiving(backup_dir, self.instance_replica, replica, replica=True) - replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - - replica.append_conf('recovery.conf', "standby_mode = 'on'") - replica.append_conf('recovery.conf', - "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port)) - replica.start({"-t": "600"}) - - # Check replica - after = replica.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Make FULL ARCHIVE backup from replica - self.add_instance(backup_dir, self.instance_replica, replica) - self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in - self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[ - '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) - self.validate_pb(backup_dir, self.instance_replica) - self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) - - # Drop Table t_heap - after = master.execute("postgres", "drop table t_heap") - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,512) i") - before = master.execute("postgres", "SELECT * FROM t_heap") - - # Make page backup from replica - self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in - self.backup_node(backup_dir, self.instance_replica, replica, backup_type='page', options=[ - '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) - self.validate_pb(backup_dir, self.instance_replica) - self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) - - @unittest.skip("skip") - def test_replica_archive_full_backup_123(self): - """ - make full archive backup from replica - """ - fname = self.id().split('.')[3] - master = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/master".format(fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) - master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.start() - - replica = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/replica".format(fname)) - replica_port = replica.port - replica.cleanup() - - self.assertEqual(self.init_pb(master), six.b("")) - self.backup_pb(node=master, backup_type='full', options=['--stream']) - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") - - before = master.execute("postgres", "SELECT * FROM t_heap") - - id = self.backup_pb(master, backup_type='page', options=['--stream']) - self.restore_pb(backup_dir=self.backup_dir(master), data_dir=replica.data_dir) - - # Settings for Replica - replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - # Set Archiving for replica - self.set_archiving_conf(replica, replica=True) - - replica.append_conf('recovery.conf', "standby_mode = 'on'") - replica.append_conf('recovery.conf', - "primary_conninfo = 'user=gsmol port={0} sslmode=prefer sslcompression=1'".format(master.port)) - replica.start({"-t": "600"}) - # Replica Started - - # master.execute("postgres", "checkpoint") - - # Check replica - after = replica.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Make backup from replica - self.assertEqual(self.init_pb(replica), six.b("")) - self.backup_pb(replica, backup_type='full', options=['--archive-timeout=30']) - self.validate_pb(replica) diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 0880c031..54d7f8a4 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -1,8 +1,6 @@ -import unittest import os -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,9 +8,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_clean' - def teardown(self): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_clean(self): @@ -89,8 +84,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - print self.show_pb(backup_dir, 'node', as_text=True) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index ff0fb6a2..13ea7678 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -1,8 +1,6 @@ -import unittest import os -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_cluster' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_cluster_btree(self): @@ -72,7 +66,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_ptrack_cluster_spgist(self): fname = self.id().split('.')[3] @@ -130,7 +125,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_ptrack_cluster_brin(self): fname = self.id().split('.')[3] @@ -188,7 +184,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_ptrack_cluster_gist(self): fname = self.id().split('.')[3] @@ -246,7 +243,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_ptrack_cluster_gin(self): fname = self.id().split('.')[3] @@ -304,7 +302,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index e35234a6..41a310eb 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -1,10 +1,6 @@ -import unittest -from sys import exit -from testgres import get_new_node, stop_all import os -from signal import SIGTERM -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack -from time import sleep +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -12,10 +8,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_move_to_tablespace' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_recovery(self): @@ -58,7 +50,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index 24802697..abc7a751 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -1,20 +1,18 @@ +import os import unittest from sys import exit -from testgres import get_new_node, stop_all -import os -from signal import SIGTERM -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack -from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) - self.module_name = 'ptrack_move_to_tablespace' + self.module_name = 'ptrack_recovery' - def teardown(self): - # clean_all() - stop_all() + # @classmethod + # def tearDownClass(cls): + # clean_all() + # shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True) # @unittest.skip("skip") # @unittest.expectedFailure @@ -45,12 +43,13 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # get path to heap and index files idx_ptrack[i]['path'] = self.get_fork_path(node, i) - print 'Killing postmaster. Losing Ptrack changes' + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) if not node.status(): node.start() else: - print "Die! Die! Why won't you die?... Why won't you die?" + print("Die! Die! Why won't you die?... Why won't you die?") exit(1) for i in idx_ptrack: @@ -60,7 +59,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index f6b22b97..fa8d145f 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -1,8 +1,6 @@ -import unittest import os -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum(self): @@ -78,7 +72,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index 1a4d3fe5..e41bb265 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -1,8 +1,6 @@ import os import unittest -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum_bits_frozen' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_frozen(self): @@ -69,7 +63,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py index ca5db705..a52325d7 100644 --- a/tests/ptrack_vacuum_bits_visibility.py +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -1,8 +1,6 @@ import os import unittest -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum_bits_visibility' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_visibility(self): @@ -69,7 +63,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index 9d9d5051..592821a0 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -1,9 +1,6 @@ import os import unittest -from sys import exit -from testgres import get_new_node, stop_all -#import os -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -11,10 +8,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum_full' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full(self): @@ -72,7 +65,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity, the most important part self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 37dd9920..3bd54b69 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -1,8 +1,6 @@ import os import unittest -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum_truncate' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate(self): @@ -71,7 +65,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/replica.py b/tests/replica.py index 9f3e90f8..b74afaad 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1,13 +1,8 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit, _getframe -import shutil -import time class ReplicaTest(ProbackupTest, unittest.TestCase): @@ -16,10 +11,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): super(ReplicaTest, self).__init__(*args, **kwargs) self.module_name = 'replica' -# @classmethod -# def tearDownClass(cls): -# stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_replica_stream_full_backup(self): @@ -65,16 +56,18 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # Make backup from replica self.add_instance(backup_dir, 'slave', slave) - #time.sleep(2) self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in self.backup_node(backup_dir, 'slave', slave, options=['--stream', '--log-level=verbose', '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)])) self.validate_pb(backup_dir, 'slave') self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status']) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + # @unittest.skip("skip") def test_replica_archive_full_backup(self): - """make full archive backup from replica""" + """make full archive backup from replica, set replica, make backup from replica""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), @@ -127,3 +120,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'slave', slave, options=['--archive-timeout=300', '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)]) self.validate_pb(backup_dir, 'slave') + self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status']) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/restore_test.py b/tests/restore_test.py index 1ef35c7d..4715e2a7 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1,12 +1,8 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import stop_all +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess from datetime import datetime -import shutil -from sys import exit class RestoreTest(ProbackupTest, unittest.TestCase): @@ -15,10 +11,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase): super(RestoreTest, self).__init__(*args, **kwargs) self.module_name = 'restore' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_restore_full_to_latest(self): @@ -45,7 +37,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.cleanup() # 1 - Test recovery from latest - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -58,7 +50,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_page_to_latest(self): @@ -89,7 +82,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -98,10 +91,11 @@ class RestoreTest(ProbackupTest, unittest.TestCase): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) - # @unittest.skip("skip") - def test_restore_to_timeline(self): + #@unittest.skip("skip") + def test_restore_to_specific_timeline(self): """recovery to target timeline""" fname = self.id().split('.')[3] node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), @@ -120,11 +114,11 @@ class RestoreTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node(backup_dir, 'node', node) - target_tli = int(node.get_control_data()[six.b("Latest checkpoint's TimeLineID")]) + target_tli = int(node.get_control_data()["Latest checkpoint's TimeLineID"]) node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -134,13 +128,13 @@ class RestoreTest(ProbackupTest, unittest.TestCase): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type="full") + self.backup_node(backup_dir, 'node', node) node.stop({"-m": "immediate"}) node.cleanup() # Correct Backup must be choosen for restore - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", "--timeline={0}".format(target_tli)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -152,7 +146,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_to_time(self): @@ -181,7 +176,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(target_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -190,7 +185,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_to_xid(self): @@ -235,7 +231,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop({"-m": "fast"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--xid={0}'.format(target_xid)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -244,7 +240,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_ptrack_archive(self): @@ -275,7 +272,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -284,7 +281,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_ptrack(self): @@ -321,7 +319,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -330,7 +328,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_ptrack_stream(self): @@ -362,7 +361,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -371,7 +370,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_ptrack_under_load(self): @@ -410,7 +410,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -418,10 +418,10 @@ class RestoreTest(ProbackupTest, unittest.TestCase): bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history") - self.assertEqual(bbalance, delta) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_under_load_ptrack(self): @@ -463,16 +463,17 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.cleanup() #self.wrong_wal_clean(node, wal_segment_size) - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history") - self.assertEqual(bbalance, delta) - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_to_xid_inclusive(self): @@ -501,9 +502,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): before = node.execute("postgres", "SELECT * FROM pgbench_branches") with node.connect("postgres") as con: - res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() - target_xid = res[0][0] + target_xid = result[0][0] pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() @@ -517,7 +518,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop({"-m": "fast"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--xid={0}'.format(target_xid), "--inclusive=false"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -528,7 +529,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.assertEqual(before, after) self.assertEqual(len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_with_tablespace_mapping_1(self): @@ -556,7 +558,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): con.commit() backup_id = self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") # 1 - Try to restore to existing directory node.stop() @@ -565,7 +567,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because restore destionation is not empty.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -577,20 +579,20 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because restore tablespace destination is not empty.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # 3 - Restore using tablespace-mapping tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() - res = node.execute("postgres", "SELECT id FROM test") - self.assertEqual(res[0][0], 1) + result = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(result[0][0], 1) # 4 - Restore using tablespace-mapping using page backup self.backup_node(backup_dir, 'node', node) @@ -600,22 +602,23 @@ class RestoreTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") show_pb = self.show_pb(backup_dir, 'node') - self.assertEqual(show_pb[1]['Status'], six.b("OK")) - self.assertEqual(show_pb[2]['Status'], six.b("OK")) + self.assertEqual(show_pb[1]['Status'], "OK") + self.assertEqual(show_pb[2]['Status'], "OK") node.stop() node.cleanup() tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() - res = node.execute("postgres", "SELECT id FROM test OFFSET 1") - self.assertEqual(res[0][0], 2) + result = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(result[0][0], 2) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_with_tablespace_mapping_2(self): @@ -633,7 +636,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Full backup self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") # Create tablespace tblspc_path = os.path.join(node.base_dir, "tblspc") @@ -647,8 +650,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # First page backup self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], six.b("PAGE")) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], "PAGE") # Create tablespace table with node.connect("postgres") as con: @@ -661,15 +664,15 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Second page backup backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], six.b("PAGE")) + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], "PAGE") node.stop() node.cleanup() tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() @@ -678,9 +681,11 @@ class RestoreTest(ProbackupTest, unittest.TestCase): self.assertEqual(count[0][0], 4) count = node.execute("postgres", "SELECT count(*) FROM tbl1") self.assertEqual(count[0][0], 4) - node.stop() - # @unittest.skip("skip") + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + #@unittest.skip("skip") def test_archive_node_backup_stream_restore_to_recovery_time(self): """make node with archiving, make stream backup, make PITR to Recovery Time""" fname = self.id().split('.')[3] @@ -692,6 +697,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -700,18 +706,57 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node.stop() node.cleanup() - recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time'] + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) - res = node.psql("postgres", 'select * from t_heap') - self.assertEqual(True, 'does not exist' in res[2]) + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) self.assertEqual(True, node.status()) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + + #@unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """make node with archiving, make stream backup, make PITR to Recovery Time""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + node.psql("postgres", "create table t_heap(a int)") + node.psql("postgres", "select pg_switch_xlog()") node.stop() + node.cleanup() + + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + node.start({"-t": "600"}) + + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + self.assertEqual(True, node.status()) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_archive_node_backup_stream_pitr(self): @@ -734,16 +779,18 @@ class RestoreTest(ProbackupTest, unittest.TestCase): recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time'] - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) - res = node.psql("postgres", 'select * from t_heap') - self.assertEqual(True, 'does not exist' in res[2]) - node.stop() + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_archive_node_backup_archive_pitr_2(self): @@ -766,13 +813,15 @@ class RestoreTest(ProbackupTest, unittest.TestCase): recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) - res = node.psql("postgres", 'select * from t_heap') - self.assertEqual(True, 'does not exist' in res[2]) - node.stop() + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/retention_test.py b/tests/retention_test.py index 8be47b6f..d7702793 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -1,9 +1,7 @@ -import unittest import os +import unittest from datetime import datetime, timedelta -from os import path, listdir -from helpers.ptrack_helpers import ProbackupTest -from testgres import stop_all +from .helpers.ptrack_helpers import ProbackupTest class RetentionTest(ProbackupTest, unittest.TestCase): @@ -12,10 +10,6 @@ class RetentionTest(ProbackupTest, unittest.TestCase): super(RetentionTest, self).__init__(*args, **kwargs) self.module_name = 'retention' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_retention_redundancy_1(self): @@ -51,17 +45,18 @@ class RetentionTest(ProbackupTest, unittest.TestCase): min_wal = None max_wal = None for line in log.splitlines(): - if line.startswith(b"INFO: removed min WAL segment"): + if line.startswith("INFO: removed min WAL segment"): min_wal = line[31:-1] - elif line.startswith(b"INFO: removed max WAL segment"): + elif line.startswith("INFO: removed max WAL segment"): max_wal = line[31:-1] - for wal_name in listdir(os.path.join(backup_dir, 'wal', 'node')): + for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): - wal_name_b = wal_name.encode('ascii') - self.assertEqual(wal_name_b[8:] > min_wal[8:], True) - self.assertEqual(wal_name_b[8:] > max_wal[8:], True) + #wal_name_b = wal_name.encode('ascii') + self.assertEqual(wal_name[8:] > min_wal[8:], True) + self.assertEqual(wal_name[8:] > max_wal[8:], True) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("123") def test_retention_window_2(self): @@ -87,12 +82,12 @@ class RetentionTest(ProbackupTest, unittest.TestCase): # Make backup to be keeped self.backup_node(backup_dir, 'node', node) - backups = path.join(backup_dir, 'backups', 'node') + backups = os.path.join(backup_dir, 'backups', 'node') days_delta = 5 - for backup in listdir(backups): + for backup in os.listdir(backups): if backup == 'pg_probackup.conf': continue - with open(path.join(backups, backup, "backup.control"), "a") as conf: + with open(os.path.join(backups, backup, "backup.control"), "a") as conf: conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=days_delta))) days_delta -= 1 @@ -106,4 +101,5 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.delete_expired(backup_dir, 'node') self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/show_test.py b/tests/show_test.py index 37b146e4..4a3df501 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -1,9 +1,6 @@ -import unittest import os -from os import path -import six -from helpers.ptrack_helpers import ProbackupTest -from testgres import stop_all +import unittest +from .helpers.ptrack_helpers import ProbackupTest class OptionTest(ProbackupTest, unittest.TestCase): @@ -12,10 +9,6 @@ class OptionTest(ProbackupTest, unittest.TestCase): super(OptionTest, self).__init__(*args, **kwargs) self.module_name = 'show' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_show_1(self): @@ -36,8 +29,10 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node, options=["--log-level=panic"]), None ) - self.assertIn(six.b("OK"), self.show_pb(backup_dir, 'node', as_text=True)) - node.stop() + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_corrupt_2(self): @@ -57,9 +52,11 @@ class OptionTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node(backup_dir, 'node', node) # delete file which belong to backup - file = path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + file = os.path.join(backup_dir, "backups", "node", backup_id, "database", "postgresql.conf") os.remove(file) self.validate_pb(backup_dir, 'node', backup_id) - self.assertIn(six.b("CORRUPT"), self.show_pb(backup_dir, as_text=True)) - node.stop() + self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py index bef73cd7..20f823de 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1,12 +1,8 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit -import re class ValidateTest(ProbackupTest, unittest.TestCase): @@ -15,10 +11,6 @@ class ValidateTest(ProbackupTest, unittest.TestCase): super(ValidateTest, self).__init__(*args, **kwargs) self.module_name = 'validate' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_validate_wal_unreal_values(self): @@ -55,7 +47,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): after_backup_time = datetime.now().replace(second=0, microsecond=0) # Validate to real time - self.assertIn(six.b("INFO: backup validation completed successfully"), + self.assertIn("INFO: backup validation completed successfully", self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(target_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -65,7 +57,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_1)]) self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -75,7 +67,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_2)]) self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertTrue('ERROR: not enough WAL records to time' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -87,7 +79,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): target_xid = res[0][0] node.execute("postgres", "SELECT pg_switch_xlog()") - self.assertIn(six.b("INFO: backup validation completed successfully"), + self.assertIn("INFO: backup validation completed successfully", self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -97,15 +89,18 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertTrue('ERROR: not enough WAL records to xid' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Validate with backup ID - self.assertIn(six.b("INFO: backup validation completed successfully"), + self.assertIn("INFO: backup validation completed successfully", self.validate_pb(backup_dir, 'node', backup_id), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + # @unittest.skip("skip") def test_validate_corrupt_wal_1(self): """make archive node, make archive backup, corrupt all wal files, run validate, expect errors""" @@ -131,22 +126,25 @@ class ValidateTest(ProbackupTest, unittest.TestCase): wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals.sort() for wal in wals: - f = open(os.path.join(wals_dir, wal), "rb+") - f.seek(42) - f.write(six.b("blablablaadssaaaaaaaaaaaaaaa")) - f.close + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(42) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close # Simple validate try: self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertTrue('Possible WAL CORRUPTION' in e.message), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd) self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_validate_corrupt_wal_2(self): @@ -178,22 +176,25 @@ class ValidateTest(ProbackupTest, unittest.TestCase): wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals.sort() for wal in wals: - f = open(os.path.join(wals_dir, wal), "rb+") - f.seek(0) - f.write(six.b("blablabla")) - f.close + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(0) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close # Validate to xid try: self.validate_pb(backup_dir, 'node', backup_id, options=['--xid={0}'.format(target_xid)]) self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertTrue('Possible WAL CORRUPTION' in e.message), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd) self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_validate_wal_lost_segment_1(self): @@ -232,7 +233,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertIn('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore'.format( file), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -243,10 +244,12 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_validate_wal_lost_segment_2(self): @@ -291,12 +294,14 @@ class ValidateTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( self.output, self.cmd)) - except ProbackupException, e: - self.assertTrue('INFO: wait for LSN' - and 'in archived WAL segment' - and 'WARNING: could not read WAL record at' + except ProbackupException as e: + self.assertTrue('INFO: wait for LSN' in e.message + and 'in archived WAL segment' in e.message + and 'WARNING: could not read WAL record at' in e.message and 'ERROR: WAL segment "{0}" is absent\n'.format(file) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.assertEqual('ERROR', self.show_pb(backup_dir, 'node')[1]['Status'], 'Backup {0} should have STATUS "ERROR"') - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) From 378ad33033609b76a726039e85f6c0f4abbe9513 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 15 Aug 2018 15:04:43 +0300 Subject: [PATCH 05/37] PGPRO-1893: Add define FRONTEND to build Windows distro --- gen_probackup_project.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index b136907b..1752f3bf 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -127,6 +127,7 @@ sub build_pgprobackup #vvs test my $probackup = $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $probackup->AddDefine('FRONTEND'); $probackup->AddFiles( 'contrib/pg_probackup/src', 'archive.c', From 820cd0fe052f41cd44ef75aad627dbe719938289 Mon Sep 17 00:00:00 2001 From: Victor Wagner Date: Fri, 19 Oct 2018 16:33:33 +0300 Subject: [PATCH 06/37] Applied patch for zero symbols in CSV for COPY FROM command --- .gitignore | 45 + .travis.yml | 7 + COPYRIGHT | 29 + Makefile | 87 + README.md | 100 + doit.cmd | 1 + doit96.cmd | 1 + gen_probackup_project.pl | 190 ++ msvs/pg_probackup.sln | 28 + msvs/template.pg_probackup.vcxproj | 212 ++ msvs/template.pg_probackup96.vcxproj | 210 ++ msvs/template.pg_probackup_2.vcxproj | 203 ++ src/archive.c | 113 + src/backup.c | 2701 ++++++++++++++++++++++++ src/catalog.c | 915 ++++++++ src/configure.c | 490 +++++ src/data.c | 1407 ++++++++++++ src/delete.c | 464 ++++ src/dir.c | 1491 +++++++++++++ src/fetch.c | 116 + src/help.c | 605 ++++++ src/init.c | 108 + src/merge.c | 526 +++++ src/parsexlog.c | 1039 +++++++++ src/pg_probackup.c | 634 ++++++ src/pg_probackup.h | 620 ++++++ src/restore.c | 919 ++++++++ src/show.c | 500 +++++ src/status.c | 118 ++ src/util.c | 349 +++ src/utils/json.c | 134 ++ src/utils/json.h | 33 + src/utils/logger.c | 621 ++++++ src/utils/logger.h | 54 + src/utils/parray.c | 196 ++ src/utils/parray.h | 35 + src/utils/pgut.c | 2417 +++++++++++++++++++++ src/utils/pgut.h | 238 +++ src/utils/thread.c | 102 + src/utils/thread.h | 35 + src/validate.c | 354 ++++ tests/Readme.md | 24 + tests/__init__.py | 69 + tests/archive.py | 833 ++++++++ tests/auth_test.py | 391 ++++ tests/backup_test.py | 522 +++++ tests/cfs_backup.py | 1161 ++++++++++ tests/cfs_restore.py | 450 ++++ tests/cfs_validate_backup.py | 25 + tests/compression.py | 496 +++++ tests/delete_test.py | 203 ++ tests/delta.py | 1265 +++++++++++ tests/exclude.py | 164 ++ tests/expected/option_help.out | 95 + tests/expected/option_version.out | 1 + tests/false_positive.py | 333 +++ tests/helpers/__init__.py | 2 + tests/helpers/cfs_helpers.py | 91 + tests/helpers/ptrack_helpers.py | 1300 ++++++++++++ tests/init_test.py | 99 + tests/logging.py | 0 tests/merge.py | 454 ++++ tests/option_test.py | 218 ++ tests/page.py | 641 ++++++ tests/pgpro560.py | 98 + tests/pgpro589.py | 80 + tests/ptrack.py | 1600 ++++++++++++++ tests/ptrack_clean.py | 253 +++ tests/ptrack_cluster.py | 268 +++ tests/ptrack_move_to_tablespace.py | 57 + tests/ptrack_recovery.py | 58 + tests/ptrack_truncate.py | 130 ++ tests/ptrack_vacuum.py | 152 ++ tests/ptrack_vacuum_bits_frozen.py | 136 ++ tests/ptrack_vacuum_bits_visibility.py | 67 + tests/ptrack_vacuum_full.py | 140 ++ tests/ptrack_vacuum_truncate.py | 142 ++ tests/replica.py | 293 +++ tests/restore_test.py | 1243 +++++++++++ tests/retention_test.py | 178 ++ tests/show_test.py | 203 ++ tests/validate_test.py | 1730 +++++++++++++++ travis/backup_restore.sh | 66 + win32build.pl | 240 +++ win32build96.pl | 240 +++ win32build_2.pl | 219 ++ 86 files changed, 34877 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 COPYRIGHT create mode 100644 Makefile create mode 100644 README.md create mode 100644 doit.cmd create mode 100644 doit96.cmd create mode 100644 gen_probackup_project.pl create mode 100644 msvs/pg_probackup.sln create mode 100644 msvs/template.pg_probackup.vcxproj create mode 100644 msvs/template.pg_probackup96.vcxproj create mode 100644 msvs/template.pg_probackup_2.vcxproj create mode 100644 src/archive.c create mode 100644 src/backup.c create mode 100644 src/catalog.c create mode 100644 src/configure.c create mode 100644 src/data.c create mode 100644 src/delete.c create mode 100644 src/dir.c create mode 100644 src/fetch.c create mode 100644 src/help.c create mode 100644 src/init.c create mode 100644 src/merge.c create mode 100644 src/parsexlog.c create mode 100644 src/pg_probackup.c create mode 100644 src/pg_probackup.h create mode 100644 src/restore.c create mode 100644 src/show.c create mode 100644 src/status.c create mode 100644 src/util.c create mode 100644 src/utils/json.c create mode 100644 src/utils/json.h create mode 100644 src/utils/logger.c create mode 100644 src/utils/logger.h create mode 100644 src/utils/parray.c create mode 100644 src/utils/parray.h create mode 100644 src/utils/pgut.c create mode 100644 src/utils/pgut.h create mode 100644 src/utils/thread.c create mode 100644 src/utils/thread.h create mode 100644 src/validate.c create mode 100644 tests/Readme.md create mode 100644 tests/__init__.py create mode 100644 tests/archive.py create mode 100644 tests/auth_test.py create mode 100644 tests/backup_test.py create mode 100644 tests/cfs_backup.py create mode 100644 tests/cfs_restore.py create mode 100644 tests/cfs_validate_backup.py create mode 100644 tests/compression.py create mode 100644 tests/delete_test.py create mode 100644 tests/delta.py create mode 100644 tests/exclude.py create mode 100644 tests/expected/option_help.out create mode 100644 tests/expected/option_version.out create mode 100644 tests/false_positive.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/cfs_helpers.py create mode 100644 tests/helpers/ptrack_helpers.py create mode 100644 tests/init_test.py create mode 100644 tests/logging.py create mode 100644 tests/merge.py create mode 100644 tests/option_test.py create mode 100644 tests/page.py create mode 100644 tests/pgpro560.py create mode 100644 tests/pgpro589.py create mode 100644 tests/ptrack.py create mode 100644 tests/ptrack_clean.py create mode 100644 tests/ptrack_cluster.py create mode 100644 tests/ptrack_move_to_tablespace.py create mode 100644 tests/ptrack_recovery.py create mode 100644 tests/ptrack_truncate.py create mode 100644 tests/ptrack_vacuum.py create mode 100644 tests/ptrack_vacuum_bits_frozen.py create mode 100644 tests/ptrack_vacuum_bits_visibility.py create mode 100644 tests/ptrack_vacuum_full.py create mode 100644 tests/ptrack_vacuum_truncate.py create mode 100644 tests/replica.py create mode 100644 tests/restore_test.py create mode 100644 tests/retention_test.py create mode 100644 tests/show_test.py create mode 100644 tests/validate_test.py create mode 100644 travis/backup_restore.sh create mode 100644 win32build.pl create mode 100644 win32build96.pl create mode 100644 win32build_2.pl diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..02d1512a --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# Object files +*.o + +# Libraries +*.lib +*.a + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.app + +# Dependencies +.deps + +# Binaries +/pg_probackup + +# Generated by test suite +/regression.diffs +/regression.out +/results +/env +/tests/__pycache__/ +/tests/helpers/__pycache__/ +/tests/tmp_dirs/ +/tests/*pyc +/tests/helpers/*pyc + +# Extra files +/src/datapagemap.c +/src/datapagemap.h +/src/logging.h +/src/receivelog.c +/src/receivelog.h +/src/streamutil.c +/src/streamutil.h +/src/xlogreader.c +/src/walmethods.c +/src/walmethods.h diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..35b49ec5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +sudo: required + +services: +- docker + +script: +- docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 00000000..49d70472 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,29 @@ +Copyright (c) 2015-2017, Postgres Professional +Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + +Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group +Portions Copyright (c) 1994, The Regents of the University of California + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION + (NTT) nor the names of its contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..56ad1b01 --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +PROGRAM = pg_probackup +OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ + src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \ + src/pg_probackup.o src/restore.o src/show.o src/status.o \ + src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ + src/xlogreader.o src/streamutil.o src/receivelog.o \ + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ + src/utils/json.o src/utils/thread.o src/merge.o + +EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h + +INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +# !USE_PGXS +else +subdir=contrib/pg_probackup +top_builddir=../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif # USE_PGXS + +ifeq ($(top_srcdir),../..) + ifeq ($(LN_S),ln -s) + srchome=$(top_srcdir)/.. + endif +else +srchome=$(top_srcdir) +endif + +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) +OBJS += src/walmethods.o +EXTRA_CLEAN += src/walmethods.c src/walmethods.h +INCLUDES += src/walmethods.h +endif + +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src +override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) +PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} + +all: checksrcdir $(INCLUDES); + +$(PROGRAM): $(OBJS) + +src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c + rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ +src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ +src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@ +src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ +src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@ +src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.h $@ +src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ +src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ + + +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) +src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ +src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ +endif + +ifeq ($(PORTNAME), aix) + CC=xlc_r +endif + +# This rule's only purpose is to give the user instructions on how to pass +# the path to PostgreSQL source tree to the makefile. +.PHONY: checksrcdir +checksrcdir: +ifndef top_srcdir + @echo "You must have PostgreSQL source tree available to compile." + @echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir" + @echo "variable: \"make top_srcdir=\"" + @exit 1 +endif diff --git a/README.md b/README.md new file mode 100644 index 00000000..1471d648 --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# pg_probackup + +`pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. + +The utility is compatible with: +* PostgreSQL 9.5, 9.6, 10; + +`PTRACK` backup support provided via following options: +* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) +* Postgres Pro Standard 9.5, 9.6 +* Postgres Pro Enterprise + +As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: +* Choosing between full and page-level incremental backups to speed up backup and recovery +* Implementing a single backup strategy for multi-server PostgreSQL clusters +* Automatic data consistency checks and on-demand backup validation without actual data recovery +* Managing backups in accordance with retention policy +* Running backup, restore, and validation processes on multiple parallel threads +* Storing backup data in a compressed state to save disk space +* Taking backups from a standby server to avoid extra load on the master server +* Extended logging settings +* Custom commands to simplify WAL log archiving + +To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. + +Using `pg_probackup`, you can take full or incremental backups: +* `Full` backups contain all the data files required to restore the database cluster from scratch. +* `Incremental` backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. `pg_probackup` supports the following modes of incremental backups: + * `PAGE` backup. In this mode, `pg_probackup` scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. + * `DELTA` backup. In this mode, `pg_probackup` read all data files in PGDATA directory and only those pages, that where changed since previous backup, are copied. Continuous archiving is not necessary for it to operate. Also this mode could impose read-only I/O pressure equal to `Full` backup. + * `PTRACK` backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special `PTRACK` bitmap for this relation. As one page requires just one bit in the `PTRACK` fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. + +Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following archiving strategies: +* `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. +* `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). + +## Limitations + +`pg_probackup` currently has the following limitations: +* Creating backups from a remote server is currently not supported. +* The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-block-size) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-wal-block-size) parameters and have the same major release number. +* Microsoft Windows operating system is not supported. +* Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately. + +## Installation and Setup +### Linux Installation +```shell +#DEB Ubuntu|Debian Packages +echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list +wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update +apt-get install pg-probackup-{10,9.6,9.5} + +#DEB-SRC Packages +echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ + /etc/apt/sources.list.d/pg_probackup.list +apt-get source pg-probackup-{10,9.6,9.5} + +#RPM Centos Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#RPM RHEL Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#RPM Oracle Linux Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#SRPM Packages +yumdownloader --source pg_probackup-{10,9.6,9.5} +``` + +To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: + +```shell +make USE_PGXS=1 PG_CONFIG= top_srcdir= +``` + +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). + +## Documentation + +Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). + +## Licence + +This module available under the same license as [PostgreSQL](https://www.postgresql.org/about/licence/). + +## Feedback + +Do not hesitate to post your issues, questions and new ideas at the [issues](https://github.com/postgrespro/pg_probackup/issues) page. + +## Authors + +Postgres Professional, Moscow, Russia. + +## Credits + +`pg_probackup` utility is based on `pg_arman`, that was originally written by NTT and then developed and maintained by Michael Paquier. \ No newline at end of file diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 00000000..b46e3b36 --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 00000000..94d242c9 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl new file mode 100644 index 00000000..3ea79e96 --- /dev/null +++ b/gen_probackup_project.pl @@ -0,0 +1,190 @@ +# -*-perl-*- hey - emacs - this is a perl file +BEGIN{ +use Cwd; +use File::Basename; + +my $pgsrc=""; +if (@ARGV==1) +{ + $pgsrc = shift @ARGV; + if($pgsrc == "--help"){ + print STDERR "Usage $0 pg-source-dir \n"; + print STDERR "Like this: \n"; + print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n"; + print STDERR "May be need input this before: \n"; + print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n"; + exit 1; + } +} +else +{ + use Cwd qw(abs_path); + my $path = dirname(abs_path($0)); + chdir($path); + chdir("../.."); + $pgsrc = cwd(); +} + +chdir("$pgsrc/src/tools/msvc"); +push(@INC, "$pgsrc/src/tools/msvc"); +chdir("../../..") if (-d "../msvc" && -d "../../../src"); + +} + +use Win32; +use Carp; +use strict; +use warnings; + + +use Project; +use Solution; +use File::Copy; +use Config; +use VSObjectFactory; +use List::Util qw(first); + +use Exporter; +our (@ISA, @EXPORT_OK); +@ISA = qw(Exporter); +@EXPORT_OK = qw(Mkvcbuild); + +my $solution; +my $libpgport; +my $libpgcommon; +my $libpgfeutils; +my $postgres; +my $libpq; +my @unlink_on_exit; + + +use lib "src/tools/msvc"; + +use Mkvcbuild; + +# if (-e "src/tools/msvc/buildenv.pl") +# { +# do "src/tools/msvc/buildenv.pl"; +# } +# elsif (-e "./buildenv.pl") +# { +# do "./buildenv.pl"; +# } + +# set up the project +our $config; +do "config_default.pl"; +do "config.pl" if (-f "src/tools/msvc/config.pl"); + +# my $vcver = Mkvcbuild::mkvcbuild($config); +my $vcver = build_pgprobackup($config); + +# check what sort of build we are doing + +my $bconf = $ENV{CONFIG} || "Release"; +my $msbflags = $ENV{MSBFLAGS} || ""; +my $buildwhat = $ARGV[1] || ""; +if (uc($ARGV[0]) eq 'DEBUG') +{ + $bconf = "Debug"; +} +elsif (uc($ARGV[0]) ne "RELEASE") +{ + $buildwhat = $ARGV[0] || ""; +} + +# ... and do it +system("msbuild pg_probackup.vcxproj /verbosity:normal $msbflags /p:Configuration=$bconf" ); + + +# report status + +my $status = $? >> 8; + +exit $status; + + + +sub build_pgprobackup +{ + our $config = shift; + + chdir('../../..') if (-d '../msvc' && -d '../../../src'); + die 'Must run from root or msvc directory' + unless (-d 'src/tools/msvc' && -d 'src'); + + # my $vsVersion = DetermineVisualStudioVersion(); + my $vsVersion = '12.00'; + + $solution = CreateSolution($vsVersion, $config); + + $libpq = $solution->AddProject('libpq', 'dll', 'interfaces', + 'src/interfaces/libpq'); + $libpgfeutils = $solution->AddProject('libpgfeutils', 'lib', 'misc'); + $libpgcommon = $solution->AddProject('libpgcommon', 'lib', 'misc'); + $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); + + #vvs test + my $probackup = + $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $probackup->AddFiles( + 'contrib/pg_probackup/src', + 'archive.c', + 'backup.c', + 'catalog.c', + 'configure.c', + 'data.c', + 'delete.c', + 'dir.c', + 'fetch.c', + 'help.c', + 'init.c', + 'parsexlog.c', + 'pg_probackup.c', + 'restore.c', + 'show.c', + 'status.c', + 'util.c', + 'validate.c' + ); + $probackup->AddFiles( + 'contrib/pg_probackup/src/utils', + 'json.c', + 'logger.c', + 'parray.c', + 'pgut.c', + 'thread.c' + ); + $probackup->AddFile('src/backend/access/transam/xlogreader.c'); + $probackup->AddFiles( + 'src/bin/pg_basebackup', + 'receivelog.c', + 'streamutil.c' + ); + + if (-e 'src/bin/pg_basebackup/walmethods.c') + { + $probackup->AddFile('src/bin/pg_basebackup/walmethods.c'); + } + + $probackup->AddFile('src/bin/pg_rewind/datapagemap.c'); + + $probackup->AddFile('src/interfaces/libpq/pthread-win32.c'); + + $probackup->AddIncludeDir('src/bin/pg_basebackup'); + $probackup->AddIncludeDir('src/bin/pg_rewind'); + $probackup->AddIncludeDir('src/interfaces/libpq'); + $probackup->AddIncludeDir('src'); + $probackup->AddIncludeDir('src/port'); + + $probackup->AddIncludeDir('contrib/pg_probackup'); + $probackup->AddIncludeDir('contrib/pg_probackup/src'); + $probackup->AddIncludeDir('contrib/pg_probackup/src/utils'); + + $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + $probackup->AddLibrary('ws2_32.lib'); + + $probackup->Save(); + return $solution->{vcver}; + +} diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 00000000..2df4b404 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 00000000..46a7b2c2 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,212 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 00000000..46e019ba --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,210 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 00000000..2fc101a4 --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/archive.c b/src/archive.c new file mode 100644 index 00000000..953a6877 --- /dev/null +++ b/src/archive.c @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * archive.c: - pg_probackup specific archive commands for archive backups. + * + * + * Portions Copyright (c) 2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +#include +#include + +/* + * pg_probackup specific archive command for archive backups + * set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup + * --wal-file-path %p --wal-file-name %f', to move backups into arclog_path. + * Where archlog_path is $BACKUP_PATH/wal/system_id. + * Currently it just copies wal files to the new location. + * TODO: Planned options: list the arclog content, + * compute and validate checksums. + */ +int +do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + int64 system_id; + pgBackupConfig *config; + bool is_compress = false; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + /* verify that archive-push --instance parameter is valid */ + config = readBackupCatalogConfigFile(); + system_id = get_system_identifier(current_dir); + + if (config->pgdata == NULL) + elog(ERROR, "cannot read pg_probackup.conf for this instance"); + + if(system_id != config->system_identifier) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have SYSTEM_ID = " INT64_FORMAT " instead of " INT64_FORMAT, + wal_file_name, instance_name, config->system_identifier, system_id); + + /* Create 'archlog_path' directory. Do nothing if it already exists. */ + dir_create_dir(arclog_path, DIR_PERMISSION); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + + if (compress_alg == PGLZ_COMPRESS) + elog(ERROR, "pglz compression is not supported"); + +#ifdef HAVE_LIBZ + if (compress_alg == ZLIB_COMPRESS) + is_compress = IsXLogFileName(wal_file_name); +#endif + + push_wal_file(absolute_wal_file_path, backup_wal_file_path, is_compress, + overwrite); + elog(INFO, "pg_probackup archive-push completed successfully"); + + return 0; +} + +/* + * pg_probackup specific restore command. + * Move files from arclog_path to pgdata/wal_file_path. + */ +int +do_archive_get(char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-get from %s to %s", + backup_wal_file_path, absolute_wal_file_path); + get_wal_file(backup_wal_file_path, absolute_wal_file_path); + elog(INFO, "pg_probackup archive-get completed successfully"); + + return 0; +} diff --git a/src/backup.c b/src/backup.c new file mode 100644 index 00000000..3aa36c98 --- /dev/null +++ b/src/backup.c @@ -0,0 +1,2701 @@ +/*------------------------------------------------------------------------- + * + * backup.c: backup DB cluster, archived WAL + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" +#include "datapagemap.h" +#include "libpq/pqsignal.h" +#include "pgtar.h" +#include "receivelog.h" +#include "storage/bufpage.h" +#include "streamutil.h" +#include "utils/thread.h" + +static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ +static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; +static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; + +/* + * How long we should wait for streaming end in seconds. + * Retreived as checkpoint_timeout + checkpoint_timeout * 0.1 + */ +static uint32 stream_stop_timeout = 0; +/* Time in which we started to wait for streaming end */ +static time_t stream_stop_begin = 0; + +const char *progname = "pg_probackup"; + +/* list of files contained in backup */ +static parray *backup_files_list = NULL; + +/* We need critical section for datapagemap_add() in case of using threads */ +static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * We need to wait end of WAL streaming before execute pg_stop_backup(). + */ +typedef struct +{ + const char *basedir; + PGconn *conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} StreamThreadArg; + +static pthread_t stream_thread; +static StreamThreadArg stream_thread_arg = {"", NULL, 1}; + +static int is_ptrack_enable = false; +bool is_ptrack_support = false; +bool is_checksum_enabled = false; +bool exclusive_backup = false; + +/* Backup connections */ +static PGconn *backup_conn = NULL; +static PGconn *master_conn = NULL; +static PGconn *backup_conn_replication = NULL; + +/* PostgreSQL server version from "backup_conn" */ +static int server_version = 0; +static char server_version_str[100] = ""; + +/* Is pg_start_backup() was executed */ +static bool backup_in_progress = false; +/* Is pg_stop_backup() was sent */ +static bool pg_stop_backup_is_sent = false; + +/* + * Backup routines + */ +static void backup_cleanup(bool fatal, void *userdata); +static void backup_disconnect(bool fatal, void *userdata); + +static void *backup_files(void *arg); +static void *remote_backup_files(void *arg); + +static void do_backup_instance(void); + +static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); +static void pg_switch_wal(PGconn *conn); +static void pg_stop_backup(pgBackup *backup); +static int checkpoint_timeout(void); + +//static void backup_list_file(parray *files, const char *root, ) +static void parse_backup_filelist_filenames(parray *files, const char *root); +static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment); +static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); +static void make_pagemap_from_ptrack(parray *files); +static void *StreamLog(void *arg); + +static void get_remote_pgdata_filelist(parray *files); +static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); +static void remote_copy_file(PGconn *conn, pgFile* file); + +/* Ptrack functions */ +static void pg_ptrack_clear(void); +static bool pg_ptrack_support(void); +static bool pg_ptrack_enable(void); +static bool pg_checksum_enable(void); +static bool pg_is_in_recovery(void); +static bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid); +static char *pg_ptrack_get_and_clear(Oid tablespace_oid, + Oid db_oid, + Oid rel_oid, + size_t *result_size); +static XLogRecPtr get_last_ptrack_lsn(void); + +/* Check functions */ +static void check_server_version(void); +static void check_system_identifiers(void); +static void confirm_block_size(const char *name, int blcksz); +static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); + +#define disconnect_and_exit(code) \ + { \ + if (conn != NULL) PQfinish(conn); \ + exit(code); \ + } + +/* Fill "files" with data about all the files to backup */ +static void +get_remote_pgdata_filelist(parray *files) +{ + PGresult *res; + int resultStatus; + int i; + + backup_conn_replication = pgut_connect_replication(pgut_dbname); + + if (PQsendQuery(backup_conn_replication, "FILE_BACKUP FILELIST") == 0) + elog(ERROR,"%s: could not send replication command \"%s\": %s", + PROGRAM_NAME, "FILE_BACKUP", PQerrorMessage(backup_conn_replication)); + + res = PQgetResult(backup_conn_replication); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + resultStatus = PQresultStatus(res); + PQclear(res); + elog(ERROR, "cannot start getting FILE_BACKUP filelist: %s, result_status %d", + PQerrorMessage(backup_conn_replication), resultStatus); + } + + if (PQntuples(res) < 1) + elog(ERROR, "%s: no data returned from server", PROGRAM_NAME); + + for (i = 0; i < PQntuples(res); i++) + { + ReceiveFileList(files, backup_conn_replication, res, i); + } + + res = PQgetResult(backup_conn_replication); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(ERROR, "%s: final receive failed: %s", + PROGRAM_NAME, PQerrorMessage(backup_conn_replication)); + } + + PQfinish(backup_conn_replication); +} + +/* + * workhorse for get_remote_pgdata_filelist(). + * Parse received message into pgFile structure. + */ +static void +ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) +{ + char filename[MAXPGPATH]; + pgoff_t current_len_left = 0; + bool basetablespace; + char *copybuf = NULL; + pgFile *pgfile; + + /* What for do we need this basetablespace field?? */ + basetablespace = PQgetisnull(res, rownum, 0); + if (basetablespace) + elog(LOG,"basetablespace"); + else + elog(LOG, "basetablespace %s", PQgetvalue(res, rownum, 1)); + + res = PQgetResult(conn); + + if (PQresultStatus(res) != PGRES_COPY_OUT) + elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(conn)); + + while (1) + { + int r; + int filemode; + + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + r = PQgetCopyData(conn, ©buf, 0); + + if (r == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); + + /* end of copy */ + if (r == -1) + break; + + /* This must be the header for a new file */ + if (r != 512) + elog(ERROR, "Invalid tar block header size: %d\n", r); + + current_len_left = read_tar_number(©buf[124], 12); + + /* Set permissions on the file */ + filemode = read_tar_number(©buf[100], 8); + + /* First part of header is zero terminated filename */ + snprintf(filename, sizeof(filename), "%s", copybuf); + + pgfile = pgFileInit(filename); + pgfile->size = current_len_left; + pgfile->mode |= filemode; + + if (filename[strlen(filename) - 1] == '/') + { + /* Symbolic link or directory has size zero */ + Assert (pgfile->size == 0); + /* Ends in a slash means directory or symlink to directory */ + if (copybuf[156] == '5') + { + /* Directory */ + pgfile->mode |= S_IFDIR; + } + else if (copybuf[156] == '2') + { + /* Symlink */ +#ifndef WIN32 + pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif + } + else + elog(ERROR, "Unrecognized link indicator \"%c\"\n", + copybuf[156]); + } + else + { + /* regular file */ + pgfile->mode |= S_IFREG; + } + + parray_append(files, pgfile); + } + + if (copybuf != NULL) + PQfreemem(copybuf); +} + +/* read one file via replication protocol + * and write it to the destination subdir in 'backup_path' */ +static void +remote_copy_file(PGconn *conn, pgFile* file) +{ + PGresult *res; + char *copybuf = NULL; + char buf[32768]; + FILE *out; + char database_path[MAXPGPATH]; + char to_path[MAXPGPATH]; + bool skip_padding = false; + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + join_path_components(to_path, database_path, file->path); + + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + INIT_CRC32C(file->crc); + + /* read from stream and write to backup file */ + while (1) + { + int row_length; + int errno_tmp; + int write_buffer_size = 0; + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + row_length = PQgetCopyData(conn, ©buf, 0); + + if (row_length == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); + + if (row_length == -1) + break; + + if (!skip_padding) + { + write_buffer_size = Min(row_length, sizeof(buf)); + memcpy(buf, copybuf, write_buffer_size); + COMP_CRC32C(file->crc, buf, write_buffer_size); + + /* TODO calc checksum*/ + if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) + { + errno_tmp = errno; + /* oops */ + FIN_CRC32C(file->crc); + fclose(out); + PQfinish(conn); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + file->read_size += write_buffer_size; + } + if (file->read_size >= file->size) + { + skip_padding = true; + } + } + + res = PQgetResult(conn); + + /* File is not found. That's normal. */ + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); + } + + file->write_size = (int64) file->read_size; + FIN_CRC32C(file->crc); + + fclose(out); +} + +/* + * Take a remote backup of the PGDATA at a file level. + * Copy all directories and files listed in backup_files_list. + */ +static void * +remote_backup_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + PGconn *file_backup_conn = NULL; + + for (i = 0; i < n_backup_files_list; i++) + { + char *query_str; + PGresult *res; + char *copybuf = NULL; + pgFile *file; + int row_length; + + file = (pgFile *) parray_get(arguments->files_list, i); + + /* We have already copied all directories */ + if (S_ISDIR(file->mode)) + continue; + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + file_backup_conn = pgut_connect_replication(pgut_dbname); + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + query_str = psprintf("FILE_BACKUP FILEPATH '%s'",file->path); + + if (PQsendQuery(file_backup_conn, query_str) == 0) + elog(ERROR,"%s: could not send replication command \"%s\": %s", + PROGRAM_NAME, query_str, PQerrorMessage(file_backup_conn)); + + res = PQgetResult(file_backup_conn); + + /* File is not found. That's normal. */ + if (PQresultStatus(res) == PGRES_COMMAND_OK) + { + PQclear(res); + PQfinish(file_backup_conn); + continue; + } + + if (PQresultStatus(res) != PGRES_COPY_OUT) + { + PQclear(res); + PQfinish(file_backup_conn); + elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(file_backup_conn)); + } + + /* read the header of the file */ + row_length = PQgetCopyData(file_backup_conn, ©buf, 0); + + if (row_length == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(file_backup_conn)); + + /* end of copy TODO handle it */ + if (row_length == -1) + elog(ERROR, "Unexpected end of COPY data"); + + if(row_length != 512) + elog(ERROR, "Invalid tar block header size: %d\n", row_length); + file->size = read_tar_number(©buf[124], 12); + + /* receive the data from stream and write to backup file */ + remote_copy_file(file_backup_conn, file); + + elog(VERBOSE, "File \"%s\". Copied " INT64_FORMAT " bytes", + file->path, file->write_size); + PQfinish(file_backup_conn); + } + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Take a backup of a single postgresql instance. + * Move files from 'pgdata' to a subdirectory in 'backup_path'. + */ +static void +do_backup_instance(void) +{ + int i; + char database_path[MAXPGPATH]; + char dst_backup_path[MAXPGPATH]; + char label[1024]; + XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; + + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + backup_files_arg *threads_args; + bool backup_isok = true; + + pgBackup *prev_backup = NULL; + parray *prev_backup_filelist = NULL; + + elog(LOG, "Database backup start"); + + /* Initialize size summary */ + current.data_bytes = 0; + + /* Obtain current timeline */ + if (is_remote_backup) + { + char *sysidentifier; + TimeLineID starttli; + XLogRecPtr startpos; + + backup_conn_replication = pgut_connect_replication(pgut_dbname); + + /* Check replication prorocol connection */ + if (!RunIdentifySystem(backup_conn_replication, &sysidentifier, &starttli, &startpos, NULL)) + elog(ERROR, "Failed to send command for remote backup"); + +// TODO implement the check +// if (&sysidentifier != system_identifier) +// elog(ERROR, "Backup data directory was initialized for system id %ld, but target backup directory system id is %ld", +// system_identifier, sysidentifier); + + current.tli = starttli; + + PQfinish(backup_conn_replication); + } + else + current.tli = get_current_timeline(false); + + /* + * In incremental backup mode ensure that already-validated + * backup on current timeline exists and get its filelist. + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK || + current.backup_mode == BACKUP_MODE_DIFF_DELTA) + { + parray *backup_list; + char prev_backup_filelist_path[MAXPGPATH]; + + /* get list of backups already taken */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + prev_backup = catalog_get_last_data_backup(backup_list, current.tli); + if (prev_backup == NULL) + elog(ERROR, "Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one."); + parray_free(backup_list); + + pgBackupGetPath(prev_backup, prev_backup_filelist_path, + lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); + /* Files of previous backup needed by DELTA backup */ + prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path); + + /* If lsn is not NULL, only pages with higher lsn will be copied. */ + prev_backup_start_lsn = prev_backup->start_lsn; + current.parent_backup = prev_backup->start_time; + + pgBackupWriteBackupControlFile(¤t); + } + + /* + * It`s illegal to take PTRACK backup if LSN from ptrack_control() is not equal to + * stort_backup LSN of previous backup + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(); + + if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) + { + elog(ERROR, "LSN from ptrack_control " UINT64_FORMAT " differs from STOP LSN of previous backup " + UINT64_FORMAT ".\n" + "Create new full backup before an incremental one.", + ptrack_lsn, prev_backup->stop_lsn); + } + } + + /* Clear ptrack files for FULL and PAGE backup */ + if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && is_ptrack_enable) + pg_ptrack_clear(); + + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time); + strncat(label, " with pg_probackup", lengthof(label) - + strlen(" with pg_probackup")); + pg_start_backup(label, smooth_checkpoint, ¤t); + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + /* start stream replication */ + if (stream_wal) + { + join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); + dir_create_dir(dst_backup_path, DIR_PERMISSION); + + stream_thread_arg.basedir = dst_backup_path; + + /* + * Connect in replication mode to the server. + */ + stream_thread_arg.conn = pgut_connect_replication(pgut_dbname); + + if (!CheckServerVersionForStreaming(stream_thread_arg.conn)) + { + PQfinish(stream_thread_arg.conn); + /* + * Error message already written in CheckServerVersionForStreaming(). + * There's no hope of recovering from a version mismatch, so don't + * retry. + */ + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* + * Identify server, obtaining start LSN position and current timeline ID + * at the same time, necessary if not valid data can be found in the + * existing output directory. + */ + if (!RunIdentifySystem(stream_thread_arg.conn, NULL, NULL, NULL, NULL)) + { + PQfinish(stream_thread_arg.conn); + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* By default there are some error */ + stream_thread_arg.ret = 1; + + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); + } + + /* initialize backup list */ + backup_files_list = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + if (is_remote_backup) + get_remote_pgdata_filelist(backup_files_list); + else + dir_list_file(backup_files_list, pgdata, true, true, false); + + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + + /* Extract information about files in backup_list parsing their names:*/ + parse_backup_filelist_filenames(backup_files_list, pgdata); + + if (current.backup_mode != BACKUP_MODE_FULL) + { + elog(LOG, "current_tli:%X", current.tli); + elog(LOG, "prev_backup->start_lsn: %X/%X", + (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn)); + elog(LOG, "current.start_lsn: %X/%X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + } + + /* + * Build page mapping in incremental mode. + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + { + /* + * Build the page map. Obtain information about changed pages + * reading WAL segments present in archives up to the point + * where this backup has started. + */ + extractPageMap(arclog_path, current.tli, xlog_seg_size, + prev_backup->start_lsn, current.start_lsn, + /* + * For backup from master wait for previous segment. + * For backup from replica wait for current segment. + */ + !current.from_replica, backup_files_list); + } + else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + /* + * Build the page map from ptrack information. + */ + make_pagemap_from_ptrack(backup_files_list); + } + + /* + * Make directories before backup and setup threads at the same time + */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + /* if the entry was a directory, create it in the backup */ + if (S_ISDIR(file->mode)) + { + char dirpath[MAXPGPATH]; + char *dir_name; + char database_path[MAXPGPATH]; + + if (!is_remote_backup) + dir_name = GetRelativePath(file->path, pgdata); + else + dir_name = file->path; + + elog(VERBOSE, "Create directory \"%s\"", dir_name); + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + join_path_components(dirpath, database_path, dir_name); + dir_create_dir(dirpath, DIR_PERMISSION); + } + + /* setup threads */ + pg_atomic_clear_flag(&file->lock); + } + + /* Sort by size for load balancing */ + parray_qsort(backup_files_list, pgFileCompareSize); + /* Sort the array for binary search */ + if (prev_backup_filelist) + parray_qsort(prev_backup_filelist, pgFileComparePath); + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); + + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + arg->from_root = pgdata; + arg->to_root = database_path; + arg->files_list = backup_files_list; + arg->prev_filelist = prev_backup_filelist; + arg->prev_start_lsn = prev_backup_start_lsn; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; + /* By default there are some error */ + arg->ret = 1; + } + + /* Run threads */ + elog(LOG, "Start transfering data files"); + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + elog(VERBOSE, "Start thread num: %i", i); + + if (!is_remote_backup) + pthread_create(&threads[i], NULL, backup_files, arg); + else + pthread_create(&threads[i], NULL, remote_backup_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + backup_isok = false; + } + if (backup_isok) + elog(LOG, "Data files are transfered"); + else + elog(ERROR, "Data files transferring failed"); + + /* clean previous backup file list */ + if (prev_backup_filelist) + { + parray_walk(prev_backup_filelist, pgFileFree); + parray_free(prev_backup_filelist); + } + + /* Notify end of backup */ + pg_stop_backup(¤t); + + /* Add archived xlog files into the list of files of this backup */ + if (stream_wal) + { + parray *xlog_files_list; + char pg_xlog_path[MAXPGPATH]; + + /* Scan backup PG_XLOG_DIR */ + xlog_files_list = parray_new(); + join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + + for (i = 0; i < parray_num(xlog_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + + if (S_ISREG(file->mode)) + calc_file_checksum(file); + /* Remove file path root prefix*/ + if (strstr(file->path, database_path) == file->path) + { + char *ptr = file->path; + + file->path = pstrdup(GetRelativePath(ptr, database_path)); + free(ptr); + } + } + + /* Add xlog files into the list of backed up files */ + parray_concat(backup_files_list, xlog_files_list); + parray_free(xlog_files_list); + } + + /* Print the list of files to backup catalog */ + pgBackupWriteFileList(¤t, backup_files_list, pgdata); + + /* Compute summary of size of regular files in the backup */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + if (S_ISDIR(file->mode)) + current.data_bytes += 4096; + + /* Count the amount of the data actually copied */ + if (S_ISREG(file->mode)) + current.data_bytes += file->write_size; + } + + parray_walk(backup_files_list, pgFileFree); + parray_free(backup_files_list); + backup_files_list = NULL; +} + +/* + * Entry point of pg_probackup BACKUP subcommand. + */ +int +do_backup(time_t start_time) +{ + + /* PGDATA and BACKUP_MODE are always required */ + if (pgdata == NULL) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); + if (current.backup_mode == BACKUP_MODE_INVALID) + elog(ERROR, "required parameter not specified: BACKUP_MODE " + "(-b, --backup-mode)"); + + /* Create connection for PostgreSQL */ + backup_conn = pgut_connect(pgut_dbname); + pgut_atexit_push(backup_disconnect, NULL); + + current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + +#if PG_VERSION_NUM >= 110000 + if (!RetrieveWalSegSize(backup_conn)) + elog(ERROR, "Failed to retreive wal_segment_size"); +#endif + + current.compress_alg = compress_alg; + current.compress_level = compress_level; + + /* Confirm data block size and xlog block size are compatible */ + confirm_block_size("block_size", BLCKSZ); + confirm_block_size("wal_block_size", XLOG_BLCKSZ); + + current.from_replica = pg_is_in_recovery(); + + /* Confirm that this server version is supported */ + check_server_version(); + + /* TODO fix it for remote backup*/ + if (!is_remote_backup) + current.checksum_version = get_data_checksum_version(true); + + is_checksum_enabled = pg_checksum_enable(); + + if (is_checksum_enabled) + elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " + "Data block corruption will be detected"); + else + elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " + "pg_probackup have no way to detect data block corruption without them. " + "Reinitialize PGDATA with option '--data-checksums'."); + + StrNCpy(current.server_version, server_version_str, + sizeof(current.server_version)); + current.stream = stream_wal; + + is_ptrack_support = pg_ptrack_support(); + if (is_ptrack_support) + { + is_ptrack_enable = pg_ptrack_enable(); + } + + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + if (!is_ptrack_support) + elog(ERROR, "This PostgreSQL instance does not support ptrack"); + else + { + if(!is_ptrack_enable) + elog(ERROR, "Ptrack is disabled"); + } + } + + if (current.from_replica) + { + /* Check master connection options */ + if (master_host == NULL) + elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); + + /* Create connection to master server */ + master_conn = pgut_connect_extended(master_host, master_port, master_db, master_user); + } + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + */ + /* TODO fix it for remote backup */ + if (!is_remote_backup) + check_system_identifiers(); + + + /* Start backup. Update backup status. */ + current.status = BACKUP_STATUS_RUNNING; + current.start_time = start_time; + + /* Create backup directory and BACKUP_CONTROL_FILE */ + if (pgBackupCreateDir(¤t)) + elog(ERROR, "cannot create backup directory"); + pgBackupWriteBackupControlFile(¤t); + + elog(LOG, "Backup destination is initialized"); + + /* set the error processing function for the backup process */ + pgut_atexit_push(backup_cleanup, NULL); + + /* backup data */ + do_backup_instance(); + pgut_atexit_pop(backup_cleanup, NULL); + + /* compute size of wal files of this backup stored in the archive */ + if (!current.stream) + { + current.wal_bytes = xlog_seg_size * + (current.stop_lsn / xlog_seg_size - + current.start_lsn / xlog_seg_size + 1); + } + + /* Backup is done. Update backup status */ + current.end_time = time(NULL); + current.status = BACKUP_STATUS_DONE; + pgBackupWriteBackupControlFile(¤t); + + //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", + // current.data_bytes); + + pgBackupValidate(¤t); + + elog(INFO, "Backup %s completed", base36enc(current.start_time)); + + /* + * After successfil backup completion remove backups + * which are expired according to retention policies + */ + if (delete_expired || delete_wal) + do_retention_purge(); + + return 0; +} + +/* + * Confirm that this server version is supported + */ +static void +check_server_version(void) +{ + PGresult *res; + + /* confirm server version */ + server_version = PQserverVersion(backup_conn); + + if (server_version == 0) + elog(ERROR, "Unknown server version %d", server_version); + + if (server_version < 100000) + sprintf(server_version_str, "%d.%d", + server_version / 10000, + (server_version / 100) % 100); + else + sprintf(server_version_str, "%d", + server_version / 10000); + + if (server_version < 90500) + elog(ERROR, + "server version is %s, must be %s or higher", + server_version_str, "9.5"); + + if (current.from_replica && server_version < 90600) + elog(ERROR, + "server version is %s, must be %s or higher for backup from replica", + server_version_str, "9.6"); + + res = pgut_execute_extended(backup_conn, "SELECT pgpro_edition()", + 0, NULL, true, true); + + /* + * Check major version of connected PostgreSQL and major version of + * compiled PostgreSQL. + */ +#ifdef PGPRO_VERSION + if (PQresultStatus(res) == PGRES_FATAL_ERROR) + /* It seems we connected to PostgreSQL (not Postgres Pro) */ + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with PostgreSQL %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str); + else if (strcmp(server_version_str, PG_MAJORVERSION) != 0 && + strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, + server_version_str, PQgetvalue(res, 0, 0)); +#else + if (PQresultStatus(res) != PGRES_FATAL_ERROR) + /* It seems we connected to Postgres Pro (not PostgreSQL) */ + elog(ERROR, "%s was built with PostgreSQL %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, + server_version_str, PQgetvalue(res, 0, 0)); + else if (strcmp(server_version_str, PG_MAJORVERSION) != 0) + elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", + PROGRAM_NAME, PG_MAJORVERSION, server_version_str); +#endif + + PQclear(res); + + /* Do exclusive backup only for PostgreSQL 9.5 */ + exclusive_backup = server_version < 90600 || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK; +} + +/* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + * All system identifiers must be equal. + */ +static void +check_system_identifiers(void) +{ + uint64 system_id_conn; + uint64 system_id_pgdata; + + system_id_pgdata = get_system_identifier(pgdata); + system_id_conn = get_remote_system_identifier(backup_conn); + + if (system_id_conn != system_identifier) + elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT + ", but connected instance system id is " UINT64_FORMAT, + system_identifier, system_id_conn); + if (system_id_pgdata != system_identifier) + elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT + ", but target backup directory system id is " UINT64_FORMAT, + system_identifier, system_id_pgdata); +} + +/* + * Ensure that target backup database is initialized with + * compatible settings. Currently check BLCKSZ and XLOG_BLCKSZ. + */ +static void +confirm_block_size(const char *name, int blcksz) +{ + PGresult *res; + char *endp; + int block_size; + + res = pgut_execute(backup_conn, "SELECT pg_catalog.current_setting($1)", 1, &name); + if (PQntuples(res) != 1 || PQnfields(res) != 1) + elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); + + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); + if ((endp && *endp) || block_size != blcksz) + elog(ERROR, + "%s(%d) is not compatible(%d expected)", + name, block_size, blcksz); + + PQclear(res); +} + +/* + * Notify start of backup to PostgreSQL server. + */ +static void +pg_start_backup(const char *label, bool smooth, pgBackup *backup) +{ + PGresult *res; + const char *params[2]; + uint32 xlogid; + uint32 xrecoff; + PGconn *conn; + + params[0] = label; + + /* For replica we call pg_start_backup() on master */ + conn = (backup->from_replica) ? master_conn : backup_conn; + + /* 2nd argument is 'fast'*/ + params[1] = smooth ? "false" : "true"; + if (!exclusive_backup) + res = pgut_execute(conn, + "SELECT pg_catalog.pg_start_backup($1, $2, false)", + 2, + params); + else + res = pgut_execute(conn, + "SELECT pg_catalog.pg_start_backup($1, $2)", + 2, + params); + + /* + * Set flag that pg_start_backup() was called. If an error will happen it + * is necessary to call pg_stop_backup() in backup_cleanup(). + */ + backup_in_progress = true; + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + PQclear(res); + + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + /* + * Switch to a new WAL segment. It is necessary to get archived WAL + * segment, which includes start LSN of current backup. + */ + pg_switch_wal(conn); + + if (!stream_wal) + { + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + /* In PAGE mode wait for current segment... */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + wait_wal_lsn(backup->start_lsn, false); + /* ...for others wait for previous segment */ + else + wait_wal_lsn(backup->start_lsn, true); + } + + /* Wait for start_lsn to be replayed by replica */ + if (backup->from_replica) + wait_replica_wal_lsn(backup->start_lsn, true); +} + +/* + * Switch to a new WAL segment. It should be called only for master. + */ +static void +pg_switch_wal(PGconn *conn) +{ + PGresult *res; + + /* Remove annoying NOTICE messages generated by backend */ + res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); + PQclear(res); + + if (server_version >= 100000) + res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_wal()", 0, NULL); + else + res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_xlog()", 0, NULL); + + PQclear(res); +} + +/* + * Check if the instance supports ptrack + * TODO Maybe we should rather check ptrack_version()? + */ +static bool +pg_ptrack_support(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, + "SELECT proname FROM pg_proc WHERE proname='ptrack_version'", + 0, NULL); + if (PQntuples(res_db) == 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + + res_db = pgut_execute(backup_conn, + "SELECT pg_catalog.ptrack_version()", + 0, NULL); + if (PQntuples(res_db) == 0) + { + PQclear(res_db); + return false; + } + + /* Now we support only ptrack versions upper than 1.5 */ + if (strcmp(PQgetvalue(res_db, 0, 0), "1.5") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0) + { + elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", PQgetvalue(res_db, 0, 0)); + PQclear(res_db); + return false; + } + + PQclear(res_db); + return true; +} + +/* Check if ptrack is enabled in target instance */ +static bool +pg_ptrack_enable(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show ptrack_enable", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* Check if ptrack is enabled in target instance */ +static bool +pg_checksum_enable(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show data_checksums", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* Check if target instance is replica */ +static bool +pg_is_in_recovery(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "SELECT pg_catalog.pg_is_in_recovery()", 0, NULL); + + if (PQgetvalue(res_db, 0, 0)[0] == 't') + { + PQclear(res_db); + return true; + } + PQclear(res_db); + return false; +} + +/* Clear ptrack files in all databases of the instance we connected to */ +static void +pg_ptrack_clear(void) +{ + PGresult *res_db, + *res; + const char *dbname; + int i; + Oid dbOid, tblspcOid; + char *params[2]; + + params[0] = palloc(64); + params[1] = palloc(64); + res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + 0, NULL); + + for(i = 0; i < PQntuples(res_db); i++) + { + PGconn *tmp_conn; + + dbname = PQgetvalue(res_db, i, 0); + if (strcmp(dbname, "template0") == 0) + continue; + + dbOid = atoi(PQgetvalue(res_db, i, 1)); + tblspcOid = atoi(PQgetvalue(res_db, i, 2)); + + tmp_conn = pgut_connect(dbname); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + PQclear(res); + + pgut_disconnect(tmp_conn); + } + + pfree(params[0]); + pfree(params[1]); + PQclear(res_db); +} + +static bool +pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid) +{ + char *params[2]; + char *dbname; + PGresult *res_db; + PGresult *res; + bool result; + + params[0] = palloc(64); + params[1] = palloc(64); + + sprintf(params[0], "%i", dbOid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return false; + + dbname = PQgetvalue(res_db, 0, 0); + + /* Always backup all files from template0 database */ + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return true; + } + PQclear(res_db); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot perform pg_ptrack_get_and_clear_db()"); + + if (!parse_bool(PQgetvalue(res, 0, 0), &result)) + elog(ERROR, + "result of pg_ptrack_get_and_clear_db() is invalid: %s", + PQgetvalue(res, 0, 0)); + + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* Read and clear ptrack files of the target relation. + * Result is a bytea ptrack map of all segments of the target relation. + * case 1: we know a tablespace_oid, db_oid, and rel_filenode + * case 2: we know db_oid and rel_filenode (no tablespace_oid, because file in pg_default) + * case 3: we know only rel_filenode (because file in pg_global) + */ +static char * +pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, + size_t *result_size) +{ + PGconn *tmp_conn; + PGresult *res_db, + *res; + char *params[2]; + char *result; + char *val; + + params[0] = palloc(64); + params[1] = palloc(64); + + /* regular file (not in directory 'global') */ + if (db_oid != 0) + { + char *dbname; + + sprintf(params[0], "%i", db_oid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return NULL; + + dbname = PQgetvalue(res_db, 0, 0); + + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return NULL; + } + + tmp_conn = pgut_connect(dbname); + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from database \"%s\" by tablespace oid %u and relation oid %u", + dbname, tablespace_oid, rel_filenode); + PQclear(res_db); + pgut_disconnect(tmp_conn); + } + /* file in directory 'global' */ + else + { + /* + * execute ptrack_get_and_clear for relation in pg_global + * Use backup_conn, cause we can do it from any database. + */ + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from pg_global tablespace and relation oid %u", + rel_filenode); + } + + val = PQgetvalue(res, 0, 0); + + /* TODO Now pg_ptrack_get_and_clear() returns bytea ending with \x. + * It should be fixed in future ptrack releases, but till then we + * can parse it. + */ + if (strcmp("x", val+1) == 0) + { + /* Ptrack file is missing */ + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* + * Wait for target 'lsn'. + * + * If current backup started in archive mode wait for 'lsn' to be archived in + * archive 'wal' directory with WAL segment file. + * If current backup started in stream mode wait for 'lsn' to be streamed in + * 'pg_wal' directory. + * + * If 'wait_prev_segment' wait for previous segment. + */ +static void +wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment) +{ + TimeLineID tli; + XLogSegNo targetSegNo; + char wal_dir[MAXPGPATH], + wal_segment_path[MAXPGPATH]; + char wal_segment[MAXFNAMELEN]; + bool file_exists = false; + uint32 try_count = 0, + timeout; + +#ifdef HAVE_LIBZ + char gz_wal_segment_path[MAXPGPATH]; +#endif + + tli = get_current_timeline(false); + + /* Compute the name of the WAL file containig requested LSN */ + GetXLogSegNo(lsn, targetSegNo, xlog_seg_size); + if (wait_prev_segment) + targetSegNo--; + GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size); + + if (stream_wal) + { + pgBackupGetPath2(¤t, wal_dir, lengthof(wal_dir), + DATABASE_DIR, PG_XLOG_DIR); + join_path_components(wal_segment_path, wal_dir, wal_segment); + + timeout = (uint32) checkpoint_timeout(); + timeout = timeout + timeout * 0.1; + } + else + { + join_path_components(wal_segment_path, arclog_path, wal_segment); + timeout = archive_timeout; + } + + if (wait_prev_segment) + elog(LOG, "Looking for segment: %s", wal_segment); + else + elog(LOG, "Looking for LSN: %X/%X in segment: %s", (uint32) (lsn >> 32), (uint32) lsn, wal_segment); + +#ifdef HAVE_LIBZ + snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", + wal_segment_path); +#endif + + /* Wait until target LSN is archived or streamed */ + while (true) + { + if (!file_exists) + { + file_exists = fileExists(wal_segment_path); + + /* Try to find compressed WAL file */ + if (!file_exists) + { +#ifdef HAVE_LIBZ + file_exists = fileExists(gz_wal_segment_path); + if (file_exists) + elog(LOG, "Found compressed WAL segment: %s", wal_segment_path); +#endif + } + else + elog(LOG, "Found WAL segment: %s", wal_segment_path); + } + + if (file_exists) + { + /* Do not check LSN for previous WAL segment */ + if (wait_prev_segment) + return; + + /* + * A WAL segment found. Check LSN on it. + */ + if ((stream_wal && wal_contains_lsn(wal_dir, lsn, tli, + xlog_seg_size)) || + (!stream_wal && wal_contains_lsn(arclog_path, lsn, tli, + xlog_seg_size))) + /* Target LSN was found */ + { + elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); + return; + } + } + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for WAL archiving"); + try_count++; + + /* Inform user if WAL segment is absent in first attempt */ + if (try_count == 1) + { + if (wait_prev_segment) + elog(INFO, "Wait for WAL segment %s to be archived", + wal_segment_path); + else + elog(INFO, "Wait for LSN %X/%X in archived WAL segment %s", + (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); + } + + if (timeout > 0 && try_count > timeout) + { + if (file_exists) + elog(ERROR, "WAL segment %s was archived, " + "but target LSN %X/%X could not be archived in %d seconds", + wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); + /* If WAL segment doesn't exist or we wait for previous segment */ + else + elog(ERROR, + "Switched WAL segment %s could not be archived in %d seconds", + wal_segment, timeout); + } + } +} + +/* + * Wait for target 'lsn' on replica instance from master. + */ +static void +wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) +{ + uint32 try_count = 0; + + while (true) + { + PGresult *res; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr replica_lsn; + + /* + * For lsn from pg_start_backup() we need it to be replayed on replica's + * data. + */ + if (is_start_backup) + { + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_replay_lsn()", + 0, NULL); + else + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_replay_location()", + 0, NULL); + } + /* + * For lsn from pg_stop_backup() we need it only to be received by + * replica and fsync()'ed on WAL segment. + */ + else + { + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_receive_lsn()", + 0, NULL); + else + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_receive_location()", + 0, NULL); + } + + /* Extract timeline and LSN from result */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + replica_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + PQclear(res); + + /* target lsn was replicated */ + if (replica_lsn >= lsn) + break; + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for target LSN"); + try_count++; + + /* Inform user if target lsn is absent in first attempt */ + if (try_count == 1) + elog(INFO, "Wait for target LSN %X/%X to be received by replica", + (uint32) (lsn >> 32), (uint32) lsn); + + if (replica_timeout > 0 && try_count > replica_timeout) + elog(ERROR, "Target LSN %X/%X could not be recevied by replica " + "in %d seconds", + (uint32) (lsn >> 32), (uint32) lsn, + replica_timeout); + } +} + +/* + * Notify end of backup to PostgreSQL server. + */ +static void +pg_stop_backup(pgBackup *backup) +{ + PGconn *conn; + PGresult *res; + PGresult *tablespace_map_content = NULL; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr restore_lsn = InvalidXLogRecPtr; + int pg_stop_backup_timeout = 0; + char path[MAXPGPATH]; + char backup_label[MAXPGPATH]; + FILE *fp; + pgFile *file; + size_t len; + char *val = NULL; + char *stop_backup_query = NULL; + + /* + * We will use this values if there are no transactions between start_lsn + * and stop_lsn. + */ + time_t recovery_time; + TransactionId recovery_xid; + + if (!backup_in_progress) + elog(ERROR, "backup is not in progress"); + + /* For replica we call pg_stop_backup() on master */ + conn = (current.from_replica) ? master_conn : backup_conn; + + /* Remove annoying NOTICE messages generated by backend */ + res = pgut_execute(conn, "SET client_min_messages = warning;", + 0, NULL); + PQclear(res); + + /* Create restore point */ + if (backup != NULL) + { + const char *params[1]; + char name[1024]; + + if (!current.from_replica) + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + base36enc(backup->start_time)); + else + snprintf(name, lengthof(name), "pg_probackup, backup_id %s. Replica Backup", + base36enc(backup->start_time)); + params[0] = name; + + res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", + 1, params); + PQclear(res); + } + + /* + * send pg_stop_backup asynchronously because we could came + * here from backup_cleanup() after some error caused by + * postgres archive_command problem and in this case we will + * wait for pg_stop_backup() forever. + */ + + if (!pg_stop_backup_is_sent) + { + bool sent = false; + + if (!exclusive_backup) + { + /* + * Stop the non-exclusive backup. Besides stop_lsn it returns from + * pg_stop_backup(false) copy of the backup label and tablespace map + * so they can be written to disk by the caller. + */ + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; + + } + else + { + + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_stop_backup() as lsn"; + } + + sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); + pg_stop_backup_is_sent = true; + if (!sent) + elog(ERROR, "Failed to send pg_stop_backup query"); + } + + /* + * Wait for the result of pg_stop_backup(), + * but no longer than PG_STOP_BACKUP_TIMEOUT seconds + */ + if (pg_stop_backup_is_sent && !in_cleanup) + { + while (1) + { + if (!PQconsumeInput(conn) || PQisBusy(conn)) + { + pg_stop_backup_timeout++; + sleep(1); + + if (interrupted) + { + pgut_cancel(conn); + elog(ERROR, "interrupted during waiting for pg_stop_backup"); + } + + if (pg_stop_backup_timeout == 1) + elog(INFO, "wait for pg_stop_backup()"); + + /* + * If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds, + * send an interrupt. + */ + if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT) + { + pgut_cancel(conn); + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", + PG_STOP_BACKUP_TIMEOUT); + } + } + else + { + res = PQgetResult(conn); + break; + } + } + + /* Check successfull execution of pg_stop_backup() */ + if (!res) + elog(ERROR, "pg_stop backup() failed"); + else + { + switch (PQresultStatus(res)) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + break; + default: + elog(ERROR, "query failed: %s query was: %s", + PQerrorMessage(conn), stop_backup_query); + } + elog(INFO, "pg_stop backup() successfully executed"); + } + + backup_in_progress = false; + + /* Extract timeline and LSN from results of pg_stop_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 2), &xlogid, &xrecoff); + /* Calculate LSN */ + stop_backup_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + if (!XRecOffIsValid(stop_backup_lsn)) + { + stop_backup_lsn = restore_lsn; + } + + if (!XRecOffIsValid(stop_backup_lsn)) + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + + /* Write backup_label and tablespace_map */ + if (!exclusive_backup) + { + Assert(PQnfields(res) >= 4); + pgBackupGetPath(¤t, path, lengthof(path), DATABASE_DIR); + + /* Write backup_label */ + join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); + fp = fopen(backup_label, PG_BINARY_W); + if (fp == NULL) + elog(ERROR, "can't open backup label file \"%s\": %s", + backup_label, strerror(errno)); + + len = strlen(PQgetvalue(res, 0, 3)); + if (fwrite(PQgetvalue(res, 0, 3), 1, len, fp) != len || + fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "can't write backup label file \"%s\": %s", + backup_label, strerror(errno)); + + /* + * It's vital to check if backup_files_list is initialized, + * because we could get here because the backup was interrupted + */ + if (backup_files_list) + { + file = pgFileNew(backup_label, true); + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_BACKUP_LABEL_FILE); + parray_append(backup_files_list, file); + } + } + + if (sscanf(PQgetvalue(res, 0, 0), XID_FMT, &recovery_xid) != 1) + elog(ERROR, + "result of txid_snapshot_xmax() is invalid: %s", + PQgetvalue(res, 0, 0)); + if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) + elog(ERROR, + "result of current_timestamp is invalid: %s", + PQgetvalue(res, 0, 1)); + + /* Get content for tablespace_map from stop_backup results + * in case of non-exclusive backup + */ + if (!exclusive_backup) + val = PQgetvalue(res, 0, 4); + + /* Write tablespace_map */ + if (!exclusive_backup && val && strlen(val) > 0) + { + char tablespace_map[MAXPGPATH]; + + join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); + fp = fopen(tablespace_map, PG_BINARY_W); + if (fp == NULL) + elog(ERROR, "can't open tablespace map file \"%s\": %s", + tablespace_map, strerror(errno)); + + len = strlen(val); + if (fwrite(val, 1, len, fp) != len || + fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "can't write tablespace map file \"%s\": %s", + tablespace_map, strerror(errno)); + + if (backup_files_list) + { + file = pgFileNew(tablespace_map, true); + if (S_ISREG(file->mode)) + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_TABLESPACE_MAP_FILE); + parray_append(backup_files_list, file); + } + } + + if (tablespace_map_content) + PQclear(tablespace_map_content); + PQclear(res); + + if (stream_wal) + { + /* Wait for the completion of stream */ + pthread_join(stream_thread, NULL); + if (stream_thread_arg.ret == 1) + elog(ERROR, "WAL streaming failed"); + } + } + + /* Fill in fields if that is the correct end of backup. */ + if (backup != NULL) + { + char *xlog_path, + stream_xlog_path[MAXPGPATH]; + + /* Wait for stop_lsn to be received by replica */ + if (backup->from_replica) + wait_replica_wal_lsn(stop_backup_lsn, false); + /* + * Wait for stop_lsn to be archived or streamed. + * We wait for stop_lsn in stream mode just in case. + */ + wait_wal_lsn(stop_backup_lsn, false); + + if (stream_wal) + { + pgBackupGetPath2(backup, stream_xlog_path, + lengthof(stream_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); + xlog_path = stream_xlog_path; + } + else + xlog_path = arclog_path; + + backup->tli = get_current_timeline(false); + backup->stop_lsn = stop_backup_lsn; + + elog(LOG, "Getting the Recovery Time from WAL"); + + if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, + backup->start_lsn, backup->stop_lsn, + &backup->recovery_time, &backup->recovery_xid)) + { + backup->recovery_time = recovery_time; + backup->recovery_xid = recovery_xid; + } + } +} + +/* + * Retreive checkpoint_timeout GUC value in seconds. + */ +static int +checkpoint_timeout(void) +{ + PGresult *res; + const char *val; + const char *hintmsg; + int val_int; + + res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); + val = PQgetvalue(res, 0, 0); + + if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + { + PQclear(res); + if (hintmsg) + elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, + hintmsg); + else + elog(ERROR, "Invalid value of checkout_timeout %s", val); + } + + PQclear(res); + + return val_int; +} + +/* + * Notify end of backup to server when "backup_label" is in the root directory + * of the DB cluster. + * Also update backup status to ERROR when the backup is not finished. + */ +static void +backup_cleanup(bool fatal, void *userdata) +{ + /* + * Update status of backup in BACKUP_CONTROL_FILE to ERROR. + * end_time != 0 means backup finished + */ + if (current.status == BACKUP_STATUS_RUNNING && current.end_time == 0) + { + elog(WARNING, "Backup %s is running, setting its status to ERROR", + base36enc(current.start_time)); + current.end_time = time(NULL); + current.status = BACKUP_STATUS_ERROR; + pgBackupWriteBackupControlFile(¤t); + } + + /* + * If backup is in progress, notify stop of backup to PostgreSQL + */ + if (backup_in_progress) + { + elog(WARNING, "backup in progress, stop backup"); + pg_stop_backup(NULL); /* don't care stop_lsn on error case */ + } +} + +/* + * Disconnect backup connection during quit pg_probackup. + */ +static void +backup_disconnect(bool fatal, void *userdata) +{ + pgut_disconnect(backup_conn); + if (master_conn) + pgut_disconnect(master_conn); +} + +/* + * Take a backup of the PGDATA at a file level. + * Copy all directories and files listed in backup_files_list. + * If the file is 'datafile' (regular relation's main fork), read it page by page, + * verify checksum and copy. + * In incremental backup mode, copy only files or datafiles' pages changed after + * previous backup. + */ +static void * +backup_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + + /* backup a file */ + for (i = 0; i < n_backup_files_list; i++) + { + int ret; + struct stat buf; + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + + elog(VERBOSE, "Copying file: \"%s\" ", file->path); + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_backup_files_list, file->path); + + /* stat file to check its current state */ + ret = stat(file->path, &buf); + if (ret == -1) + { + if (errno == ENOENT) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + file->write_size = BYTES_INVALID; + elog(LOG, "File \"%s\" is not found", file->path); + continue; + } + else + { + elog(ERROR, + "can't stat file to backup \"%s\": %s", + file->path, strerror(errno)); + } + } + + /* We have already copied all directories */ + if (S_ISDIR(buf.st_mode)) + continue; + + if (S_ISREG(buf.st_mode)) + { + /* Check that file exist in previous backup */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + char *relative; + pgFile key; + pgFile **prev_file; + + relative = GetRelativePath(file->path, arguments->from_root); + key.path = relative; + + prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, + &key, pgFileComparePath); + if (prev_file) + /* File exists in previous backup */ + file->exists_in_prev = true; + } + /* copy the file into backup */ + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, arguments->to_root, + file->path + strlen(arguments->from_root) + 1); + + /* backup block by block if datafile AND not compressed by cfs*/ + if (!backup_data_file(arguments, to_path, file, + arguments->prev_start_lsn, + current.backup_mode, + compress_alg, compress_level)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); + continue; + } + } + /* TODO: + * Check if file exists in previous backup + * If exists: + * if mtime > start_backup_time of parent backup, + * copy file to backup + * if mtime < start_backup_time + * calculate crc, compare crc to old file + * if crc is the same -> skip file + */ + else if (!copy_file(arguments->from_root, arguments->to_root, file)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); + continue; + } + + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + file->path, file->write_size); + } + else + elog(LOG, "unexpected file type %d", buf.st_mode); + } + + /* Close connection */ + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Extract information about files in backup_list parsing their names: + * - remove temp tables from the list + * - remove unlogged tables from the list (leave the _init fork) + * - set flags for database directories + * - set flags for datafiles + */ +static void +parse_backup_filelist_filenames(parray *files, const char *root) +{ + size_t i = 0; + Oid unlogged_file_reloid = 0; + + while (i < parray_num(files)) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *relative; + int sscanf_result; + + relative = GetRelativePath(file->path, root); + + if (S_ISREG(file->mode) && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative)) + { + /* + * Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + * Legal only in case of 'pg_compression' + */ + if (strcmp(file->name, "pg_compression") == 0) + { + Oid tblspcOid; + Oid dbOid; + char tmp_rel_path[MAXPGPATH]; + /* + * Check that the file is located under + * TABLESPACE_VERSION_DIRECTORY + */ + sscanf_result = sscanf(relative, PG_TBLSPC_DIR "/%u/%s/%u", + &tblspcOid, tmp_rel_path, &dbOid); + + /* Yes, it is */ + if (sscanf_result == 2 && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + set_cfs_datafiles(files, root, relative, i); + } + } + + if (S_ISREG(file->mode) && file->tblspcOid != 0 && + file->name && file->name[0]) + { + if (strcmp(file->forkName, "init") == 0) + { + /* + * Do not backup files of unlogged relations. + * scan filelist backward and exclude these files. + */ + int unlogged_file_num = i - 1; + pgFile *unlogged_file = (pgFile *) parray_get(files, + unlogged_file_num); + + unlogged_file_reloid = file->relOid; + + while (unlogged_file_num >= 0 && + (unlogged_file_reloid != 0) && + (unlogged_file->relOid == unlogged_file_reloid)) + { + pgFileFree(unlogged_file); + parray_remove(files, unlogged_file_num); + + unlogged_file_num--; + i--; + + unlogged_file = (pgFile *) parray_get(files, + unlogged_file_num); + } + } + } + + i++; + } +} + +/* If file is equal to pg_compression, then we consider this tablespace as + * cfs-compressed and should mark every file in this tablespace as cfs-file + * Setting is_cfs is done via going back through 'files' set every file + * that contain cfs_tablespace in his path as 'is_cfs' + * Goings back through array 'files' is valid option possible because of current + * sort rules: + * tblspcOid/TABLESPACE_VERSION_DIRECTORY + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1 + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression + */ +static void +set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) +{ + int len; + int p; + pgFile *prev_file; + char *cfs_tblspc_path; + char *relative_prev_file; + + cfs_tblspc_path = strdup(relative); + if(!cfs_tblspc_path) + elog(ERROR, "Out of memory"); + len = strlen("/pg_compression"); + cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0; + elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); + + for (p = (int) i; p >= 0; p--) + { + prev_file = (pgFile *) parray_get(files, (size_t) p); + relative_prev_file = GetRelativePath(prev_file->path, root); + + elog(VERBOSE, "Checking file in cfs tablespace %s", relative_prev_file); + + if (strstr(relative_prev_file, cfs_tblspc_path) != NULL) + { + if (S_ISREG(prev_file->mode) && prev_file->is_datafile) + { + elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s", + relative_prev_file, prev_file->name); + prev_file->is_cfs = true; + } + } + else + { + elog(VERBOSE, "Breaking on %s", relative_prev_file); + break; + } + } + free(cfs_tblspc_path); +} + +/* + * Find pgfile by given rnode in the backup_files_list + * and add given blkno to its pagemap. + */ +void +process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) +{ + char *path; + char *rel_path; + BlockNumber blkno_inseg; + int segno; + pgFile **file_item; + pgFile f; + + segno = blkno / RELSEG_SIZE; + blkno_inseg = blkno % RELSEG_SIZE; + + rel_path = relpathperm(rnode, forknum); + if (segno > 0) + path = psprintf("%s/%s.%u", pgdata, rel_path, segno); + else + path = psprintf("%s/%s", pgdata, rel_path); + + pg_free(rel_path); + + f.path = path; + /* backup_files_list should be sorted before */ + file_item = (pgFile **) parray_bsearch(backup_files_list, &f, + pgFileComparePath); + + /* + * If we don't have any record of this file in the file map, it means + * that it's a relation that did not have much activity since the last + * backup. We can safely ignore it. If it is a new relation file, the + * backup would simply copy it as-is. + */ + if (file_item) + { + /* We need critical section only we use more than one threads */ + if (num_threads > 1) + pthread_lock(&backup_pagemap_mutex); + + datapagemap_add(&(*file_item)->pagemap, blkno_inseg); + + if (num_threads > 1) + pthread_mutex_unlock(&backup_pagemap_mutex); + } + + pg_free(path); +} + +/* + * Given a list of files in the instance to backup, build a pagemap for each + * data file that has ptrack. Result is saved in the pagemap field of pgFile. + * NOTE we rely on the fact that provided parray is sorted by file->path. + */ +static void +make_pagemap_from_ptrack(parray *files) +{ + size_t i; + Oid dbOid_with_ptrack_init = 0; + Oid tblspcOid_with_ptrack_init = 0; + char *ptrack_nonparsed = NULL; + size_t ptrack_nonparsed_size = 0; + + elog(LOG, "Compiling pagemap"); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + size_t start_addr; + + /* + * If there is a ptrack_init file in the database, + * we must backup all its files, ignoring ptrack files for relations. + */ + if (file->is_database) + { + char *filename = strrchr(file->path, '/'); + + Assert(filename != NULL); + filename++; + + /* + * The function pg_ptrack_get_and_clear_db returns true + * if there was a ptrack_init file. + * Also ignore ptrack files for global tablespace, + * to avoid any possible specific errors. + */ + if ((file->tblspcOid == GLOBALTABLESPACE_OID) || + pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid)) + { + dbOid_with_ptrack_init = file->dbOid; + tblspcOid_with_ptrack_init = file->tblspcOid; + } + } + + if (file->is_datafile) + { + if (file->tblspcOid == tblspcOid_with_ptrack_init && + file->dbOid == dbOid_with_ptrack_init) + { + /* ignore ptrack if ptrack_init exists */ + elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); + file->pagemap_isabsent = true; + continue; + } + + /* get ptrack bitmap once for all segments of the file */ + if (file->segno == 0) + { + /* release previous value */ + pg_free(ptrack_nonparsed); + ptrack_nonparsed_size = 0; + + ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, + file->relOid, &ptrack_nonparsed_size); + } + + if (ptrack_nonparsed != NULL) + { + /* + * pg_ptrack_get_and_clear() returns ptrack with VARHDR cutted out. + * Compute the beginning of the ptrack map related to this segment + * + * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 + * RELSEG_SIZE. Number of Pages per segment: 131072 + * RELSEG_SIZE/HEAPBLOCKS_PER_BYTE. number of bytes in ptrack file needed + * to keep track on one relsegment: 16384 + */ + start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; + + /* + * If file segment was created after we have read ptrack, + * we won't have a bitmap for this segment. + */ + if (start_addr > ptrack_nonparsed_size) + { + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + else + { + + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + { + file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + else + { + file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + + file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); + memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + } + } + else + { + /* + * If ptrack file is missing, try to copy the entire file. + * It can happen in two cases: + * - files were created by commands that bypass buffer manager + * and, correspondingly, ptrack mechanism. + * i.e. CREATE DATABASE + * - target relation was deleted. + */ + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + } + } + elog(LOG, "Pagemap compiled"); +// res = pgut_execute(backup_conn, "SET client_min_messages = warning;", 0, NULL, true); +// PQclear(pgut_execute(backup_conn, "CHECKPOINT;", 0, NULL, true)); +} + + +/* + * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is + * set by pg_stop_backup(). + */ +static bool +stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) +{ + static uint32 prevtimeline = 0; + static XLogRecPtr prevpos = InvalidXLogRecPtr; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + + /* we assume that we get called once at the end of each segment */ + if (segment_finished) + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + + /* + * Note that we report the previous, not current, position here. After a + * timeline switch, xlogpos points to the beginning of the segment because + * that's where we always begin streaming. Reporting the end of previous + * timeline isn't totally accurate, because the next timeline can begin + * slightly before the end of the WAL that we received on the previous + * timeline, but it's close enough for reporting purposes. + */ + if (prevtimeline != 0 && prevtimeline != timeline) + elog(LOG, _("switched to timeline %u at %X/%X\n"), + timeline, (uint32) (prevpos >> 32), (uint32) prevpos); + + if (!XLogRecPtrIsInvalid(stop_backup_lsn)) + { + if (xlogpos > stop_backup_lsn) + { + stop_stream_lsn = xlogpos; + return true; + } + + /* pg_stop_backup() was executed, wait for the completion of stream */ + if (stream_stop_timeout == 0) + { + elog(INFO, "Wait for LSN %X/%X to be streamed", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); + + stream_stop_timeout = checkpoint_timeout(); + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + + stream_stop_begin = time(NULL); + } + + if (time(NULL) - stream_stop_begin > stream_stop_timeout) + elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, + stream_stop_timeout); + } + + prevtimeline = timeline; + prevpos = xlogpos; + + return false; +} + +/* + * Start the log streaming + */ +static void * +StreamLog(void *arg) +{ + XLogRecPtr startpos; + TimeLineID starttli; + StreamThreadArg *stream_arg = (StreamThreadArg *) arg; + + /* + * We must use startpos as start_lsn from start_backup + */ + startpos = current.start_lsn; + starttli = current.tli; + + /* + * Always start streaming at the beginning of a segment + */ + startpos -= startpos % xlog_seg_size; + + /* Initialize timeout */ + stream_stop_timeout = 0; + stream_stop_begin = 0; + + /* + * Start the replication + */ + elog(LOG, _("started streaming WAL at %X/%X (timeline %u)"), + (uint32) (startpos >> 32), (uint32) startpos, starttli); + +#if PG_VERSION_NUM >= 90600 + { + StreamCtl ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + + ctl.startpos = startpos; + ctl.timeline = starttli; + ctl.sysidentifier = NULL; + +#if PG_VERSION_NUM >= 100000 + ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); + ctl.replication_slot = replication_slot; + ctl.stop_socket = PGINVALID_SOCKET; +#else + ctl.basedir = (char *) stream_arg->basedir; +#endif + + ctl.stream_stop = stop_streaming; + ctl.standby_message_timeout = standby_message_timeout; + ctl.partial_suffix = NULL; + ctl.synchronous = false; + ctl.mark_done = false; + + if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) + elog(ERROR, "Problem in receivexlog"); + +#if PG_VERSION_NUM >= 100000 + if (!ctl.walmethod->finish()) + elog(ERROR, "Could not finish writing WAL files: %s", + strerror(errno)); +#endif + } +#else + if(ReceiveXlogStream(stream_arg->conn, startpos, starttli, NULL, + (char *) stream_arg->basedir, stop_streaming, + standby_message_timeout, NULL, false, false) == false) + elog(ERROR, "Problem in receivexlog"); +#endif + + elog(LOG, _("finished streaming WAL at %X/%X (timeline %u)"), + (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, starttli); + stream_arg->ret = 0; + + PQfinish(stream_arg->conn); + stream_arg->conn = NULL; + + return NULL; +} + +/* + * Get lsn of the moment when ptrack was enabled the last time. + */ +static XLogRecPtr +get_last_ptrack_lsn(void) + +{ + PGresult *res; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr lsn; + + res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()", 0, NULL); + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + PQclear(res); + return lsn; +} + +char * +pg_ptrack_get_block(backup_files_arg *arguments, + Oid dbOid, + Oid tblsOid, + Oid relOid, + BlockNumber blknum, + size_t *result_size) +{ + PGresult *res; + char *params[4]; + char *result; + + params[0] = palloc(64); + params[1] = palloc(64); + params[2] = palloc(64); + params[3] = palloc(64); + + /* + * Use tmp_conn, since we may work in parallel threads. + * We can connect to any database. + */ + sprintf(params[0], "%i", tblsOid); + sprintf(params[1], "%i", dbOid); + sprintf(params[2], "%i", relOid); + sprintf(params[3], "%u", blknum); + + if (arguments->backup_conn == NULL) + { + arguments->backup_conn = pgut_connect(pgut_dbname); + } + + if (arguments->cancel_conn == NULL) + arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + + //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, + "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", + 4, (const char **)params, true); + + if (PQnfields(res) != 1) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + if (PQgetisnull(res, 0, 0)) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + + PQclear(res); + + pfree(params[0]); + pfree(params[1]); + pfree(params[2]); + pfree(params[3]); + + return result; +} diff --git a/src/catalog.c b/src/catalog.c new file mode 100644 index 00000000..f3f75277 --- /dev/null +++ b/src/catalog.c @@ -0,0 +1,915 @@ +/*------------------------------------------------------------------------- + * + * catalog.c: backup catalog operation + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; +static pgBackup *readBackupControlFile(const char *path); + +static bool exit_hook_registered = false; +static char lock_file[MAXPGPATH]; + +static void +unlink_lock_atexit(void) +{ + int res; + res = unlink(lock_file); + if (res != 0 && res != ENOENT) + elog(WARNING, "%s: %s", lock_file, strerror(errno)); +} + +/* + * Create a lockfile. + */ +void +catalog_lock(void) +{ + int fd; + char buffer[MAXPGPATH * 2 + 256]; + int ntries; + int len; + int encoded_pid; + pid_t my_pid, + my_p_pid; + + join_path_components(lock_file, backup_instance_path, BACKUP_CATALOG_PID); + + /* + * If the PID in the lockfile is our own PID or our parent's or + * grandparent's PID, then the file must be stale (probably left over from + * a previous system boot cycle). We need to check this because of the + * likelihood that a reboot will assign exactly the same PID as we had in + * the previous reboot, or one that's only one or two counts larger and + * hence the lockfile's PID now refers to an ancestor shell process. We + * allow pg_ctl to pass down its parent shell PID (our grandparent PID) + * via the environment variable PG_GRANDPARENT_PID; this is so that + * launching the postmaster via pg_ctl can be just as reliable as + * launching it directly. There is no provision for detecting + * further-removed ancestor processes, but if the init script is written + * carefully then all but the immediate parent shell will be root-owned + * processes and so the kill test will fail with EPERM. Note that we + * cannot get a false negative this way, because an existing postmaster + * would surely never launch a competing postmaster or pg_ctl process + * directly. + */ + my_pid = getpid(); +#ifndef WIN32 + my_p_pid = getppid(); +#else + + /* + * Windows hasn't got getppid(), but doesn't need it since it's not using + * real kill() either... + */ + my_p_pid = 0; +#endif + + /* + * We need a loop here because of race conditions. But don't loop forever + * (for example, a non-writable $backup_instance_path directory might cause a failure + * that won't go away). 100 tries seems like plenty. + */ + for (ntries = 0;; ntries++) + { + /* + * Try to create the lock file --- O_EXCL makes this atomic. + * + * Think not to make the file protection weaker than 0600. See + * comments below. + */ + fd = open(lock_file, O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd >= 0) + break; /* Success; exit the retry loop */ + + /* + * Couldn't create the pid file. Probably it already exists. + */ + if ((errno != EEXIST && errno != EACCES) || ntries > 100) + elog(ERROR, "could not create lock file \"%s\": %s", + lock_file, strerror(errno)); + + /* + * Read the file to get the old owner's PID. Note race condition + * here: file might have been deleted since we tried to create it. + */ + fd = open(lock_file, O_RDONLY, 0600); + if (fd < 0) + { + if (errno == ENOENT) + continue; /* race condition; try again */ + elog(ERROR, "could not open lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if ((len = read(fd, buffer, sizeof(buffer) - 1)) < 0) + elog(ERROR, "could not read lock file \"%s\": %s", + lock_file, strerror(errno)); + close(fd); + + if (len == 0) + elog(ERROR, "lock file \"%s\" is empty", lock_file); + + buffer[len] = '\0'; + encoded_pid = atoi(buffer); + + if (encoded_pid <= 0) + elog(ERROR, "bogus data in lock file \"%s\": \"%s\"", + lock_file, buffer); + + /* + * Check to see if the other process still exists + * + * Per discussion above, my_pid, my_p_pid can be + * ignored as false matches. + * + * Normally kill() will fail with ESRCH if the given PID doesn't + * exist. + */ + if (encoded_pid != my_pid && encoded_pid != my_p_pid) + { + if (kill(encoded_pid, 0) == 0 || + (errno != ESRCH && errno != EPERM)) + elog(ERROR, "lock file \"%s\" already exists", lock_file); + } + + /* + * Looks like nobody's home. Unlink the file and try again to create + * it. Need a loop because of possible race condition against other + * would-be creators. + */ + if (unlink(lock_file) < 0) + elog(ERROR, "could not remove old lock file \"%s\": %s", + lock_file, strerror(errno)); + } + + /* + * Successfully created the file, now fill it. + */ + snprintf(buffer, sizeof(buffer), "%d\n", my_pid); + + errno = 0; + if (write(fd, buffer, strlen(buffer)) != strlen(buffer)) + { + int save_errno = errno; + + close(fd); + unlink(lock_file); + /* if write didn't set errno, assume problem is no disk space */ + errno = save_errno ? save_errno : ENOSPC; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if (fsync(fd) != 0) + { + int save_errno = errno; + + close(fd); + unlink(lock_file); + errno = save_errno; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if (close(fd) != 0) + { + int save_errno = errno; + + unlink(lock_file); + errno = save_errno; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + + /* + * Arrange to unlink the lock file(s) at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(unlink_lock_atexit); + exit_hook_registered = true; + } +} + +/* + * Read backup meta information from BACKUP_CONTROL_FILE. + * If no backup matches, return NULL. + */ +pgBackup * +read_backup(time_t timestamp) +{ + pgBackup tmp; + char conf_path[MAXPGPATH]; + + tmp.start_time = timestamp; + pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); + + return readBackupControlFile(conf_path); +} + +/* + * Get backup_mode in string representation. + */ +const char * +pgBackupGetBackupMode(pgBackup *backup) +{ + return backupModes[backup->backup_mode]; +} + +static bool +IsDir(const char *dirpath, const char *entry) +{ + char path[MAXPGPATH]; + struct stat st; + + snprintf(path, MAXPGPATH, "%s/%s", dirpath, entry); + + return stat(path, &st) == 0 && S_ISDIR(st.st_mode); +} + +/* + * Create list of backups. + * If 'requested_backup_id' is INVALID_BACKUP_ID, return list of all backups. + * The list is sorted in order of descending start time. + * If valid backup id is passed only matching backup will be added to the list. + */ +parray * +catalog_get_backup_list(time_t requested_backup_id) +{ + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; + parray *backups = NULL; + pgBackup *backup = NULL; + int i; + + /* open backup instance backups directory */ + data_dir = opendir(backup_instance_path); + if (data_dir == NULL) + { + elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, + strerror(errno)); + goto err_proc; + } + + /* scan the directory and list backups */ + backups = parray_new(); + for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) + { + char backup_conf_path[MAXPGPATH]; + char data_path[MAXPGPATH]; + + /* skip not-directory entries and hidden entries */ + if (!IsDir(backup_instance_path, data_ent->d_name) + || data_ent->d_name[0] == '.') + continue; + + /* open subdirectory of specific backup */ + join_path_components(data_path, backup_instance_path, data_ent->d_name); + + /* read backup information from BACKUP_CONTROL_FILE */ + snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); + backup = readBackupControlFile(backup_conf_path); + + /* ignore corrupted backups */ + if (backup) + { + backup->backup_id = backup->start_time; + + if (requested_backup_id != INVALID_BACKUP_ID + && requested_backup_id != backup->start_time) + { + pgBackupFree(backup); + continue; + } + parray_append(backups, backup); + backup = NULL; + } + + if (errno && errno != ENOENT) + { + elog(WARNING, "cannot read data directory \"%s\": %s", + data_ent->d_name, strerror(errno)); + goto err_proc; + } + } + if (errno) + { + elog(WARNING, "cannot read backup root directory \"%s\": %s", + backup_instance_path, strerror(errno)); + goto err_proc; + } + + closedir(data_dir); + data_dir = NULL; + + parray_qsort(backups, pgBackupCompareIdDesc); + + /* Link incremental backups with their ancestors.*/ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *curr = parray_get(backups, i); + + int j; + + if (curr->backup_mode == BACKUP_MODE_FULL) + continue; + + for (j = i+1; j < parray_num(backups); j++) + { + pgBackup *ancestor = parray_get(backups, j); + + if (ancestor->start_time == curr->parent_backup) + { + curr->parent_backup_link = ancestor; + /* elog(INFO, "curr %s, ancestor %s j=%d", base36enc_dup(curr->start_time), + base36enc_dup(ancestor->start_time), j); */ + break; + } + } + } + + return backups; + +err_proc: + if (data_dir) + closedir(data_dir); + if (backup) + pgBackupFree(backup); + if (backups) + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(ERROR, "Failed to get backup list"); + + return NULL; +} + +/* + * Find the last completed backup on given timeline + */ +pgBackup * +catalog_get_last_data_backup(parray *backup_list, TimeLineID tli) +{ + int i; + pgBackup *backup = NULL; + + /* backup_list is sorted in order of descending ID */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->status == BACKUP_STATUS_OK && backup->tli == tli) + return backup; + } + + return NULL; +} + +/* create backup directory in $BACKUP_PATH */ +int +pgBackupCreateDir(pgBackup *backup) +{ + int i; + char path[MAXPGPATH]; + char *subdirs[] = { DATABASE_DIR, NULL }; + + pgBackupGetPath(backup, path, lengthof(path), NULL); + + if (!dir_is_empty(path)) + elog(ERROR, "backup destination is not empty \"%s\"", path); + + dir_create_dir(path, DIR_PERMISSION); + + /* create directories for actual backup files */ + for (i = 0; subdirs[i]; i++) + { + pgBackupGetPath(backup, path, lengthof(path), subdirs[i]); + dir_create_dir(path, DIR_PERMISSION); + } + + return 0; +} + +/* + * Write information about backup.in to stream "out". + */ +void +pgBackupWriteControl(FILE *out, pgBackup *backup) +{ + char timestamp[100]; + + fprintf(out, "#Configuration\n"); + fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); + fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fprintf(out, "compress-alg = %s\n", + deparse_compress_alg(backup->compress_alg)); + fprintf(out, "compress-level = %d\n", backup->compress_level); + fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); + + fprintf(out, "\n#Compatibility\n"); + fprintf(out, "block-size = %u\n", backup->block_size); + fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); + fprintf(out, "checksum-version = %u\n", backup->checksum_version); + fprintf(out, "program-version = %s\n", PROGRAM_VERSION); + if (backup->server_version[0] != '\0') + fprintf(out, "server-version = %s\n", backup->server_version); + + fprintf(out, "\n#Result backup info\n"); + fprintf(out, "timelineid = %d\n", backup->tli); + /* LSN returned by pg_start_backup */ + fprintf(out, "start-lsn = %X/%X\n", + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn); + /* LSN returned by pg_stop_backup */ + fprintf(out, "stop-lsn = %X/%X\n", + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + fprintf(out, "start-time = '%s'\n", timestamp); + if (backup->end_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + fprintf(out, "end-time = '%s'\n", timestamp); + } + fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + fprintf(out, "recovery-time = '%s'\n", timestamp); + } + + /* + * Size of PGDATA directory. The size does not include size of related + * WAL segments in archive 'wal' directory. + */ + if (backup->data_bytes != BYTES_INVALID) + fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + + if (backup->wal_bytes != BYTES_INVALID) + fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + + fprintf(out, "status = %s\n", status2str(backup->status)); + + /* 'parent_backup' is set if it is incremental backup */ + if (backup->parent_backup != 0) + fprintf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); + + /* print connection info except password */ + if (backup->primary_conninfo) + fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); +} + +/* create BACKUP_CONTROL_FILE */ +void +pgBackupWriteBackupControlFile(pgBackup *backup) +{ + FILE *fp = NULL; + char ini_path[MAXPGPATH]; + + pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_CONTROL_FILE); + fp = fopen(ini_path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open configuration file \"%s\": %s", ini_path, + strerror(errno)); + + pgBackupWriteControl(fp, backup); + + fclose(fp); +} + +/* + * Output the list of files to backup catalog DATABASE_FILE_LIST + */ +void +pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) +{ + FILE *fp; + char path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open file list \"%s\": %s", path, + strerror(errno)); + + print_file_list(fp, files, root); + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); +} + +/* + * Read BACKUP_CONTROL_FILE and create pgBackup. + * - Comment starts with ';'. + * - Do not care section. + */ +static pgBackup * +readBackupControlFile(const char *path) +{ + pgBackup *backup = pgut_new(pgBackup); + char *backup_mode = NULL; + char *start_lsn = NULL; + char *stop_lsn = NULL; + char *status = NULL; + char *parent_backup = NULL; + char *program_version = NULL; + char *server_version = NULL; + char *compress_alg = NULL; + int parsed_options; + + pgut_option options[] = + { + {'s', 0, "backup-mode", &backup_mode, SOURCE_FILE_STRICT}, + {'u', 0, "timelineid", &backup->tli, SOURCE_FILE_STRICT}, + {'s', 0, "start-lsn", &start_lsn, SOURCE_FILE_STRICT}, + {'s', 0, "stop-lsn", &stop_lsn, SOURCE_FILE_STRICT}, + {'t', 0, "start-time", &backup->start_time, SOURCE_FILE_STRICT}, + {'t', 0, "end-time", &backup->end_time, SOURCE_FILE_STRICT}, + {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, + {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, + {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, + {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, + {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, + {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, + {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, + {'s', 0, "program-version", &program_version, SOURCE_FILE_STRICT}, + {'s', 0, "server-version", &server_version, SOURCE_FILE_STRICT}, + {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, + {'s', 0, "status", &status, SOURCE_FILE_STRICT}, + {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, + {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, + {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, + {0} + }; + + if (access(path, F_OK) != 0) + { + elog(WARNING, "Control file \"%s\" doesn't exist", path); + pgBackupFree(backup); + return NULL; + } + + pgBackupInit(backup); + parsed_options = pgut_readopt(path, options, WARNING, true); + + if (parsed_options == 0) + { + elog(WARNING, "Control file \"%s\" is empty", path); + pgBackupFree(backup); + return NULL; + } + + if (backup->start_time == 0) + { + elog(WARNING, "Invalid ID/start-time, control file \"%s\" is corrupted", path); + pgBackupFree(backup); + return NULL; + } + + if (backup_mode) + { + backup->backup_mode = parse_backup_mode(backup_mode); + free(backup_mode); + } + + if (start_lsn) + { + uint32 xlogid; + uint32 xrecoff; + + if (sscanf(start_lsn, "%X/%X", &xlogid, &xrecoff) == 2) + backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + elog(WARNING, "Invalid START_LSN \"%s\"", start_lsn); + free(start_lsn); + } + + if (stop_lsn) + { + uint32 xlogid; + uint32 xrecoff; + + if (sscanf(stop_lsn, "%X/%X", &xlogid, &xrecoff) == 2) + backup->stop_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + elog(WARNING, "Invalid STOP_LSN \"%s\"", stop_lsn); + free(stop_lsn); + } + + if (status) + { + if (strcmp(status, "OK") == 0) + backup->status = BACKUP_STATUS_OK; + else if (strcmp(status, "ERROR") == 0) + backup->status = BACKUP_STATUS_ERROR; + else if (strcmp(status, "RUNNING") == 0) + backup->status = BACKUP_STATUS_RUNNING; + else if (strcmp(status, "MERGING") == 0) + backup->status = BACKUP_STATUS_MERGING; + else if (strcmp(status, "DELETING") == 0) + backup->status = BACKUP_STATUS_DELETING; + else if (strcmp(status, "DELETED") == 0) + backup->status = BACKUP_STATUS_DELETED; + else if (strcmp(status, "DONE") == 0) + backup->status = BACKUP_STATUS_DONE; + else if (strcmp(status, "ORPHAN") == 0) + backup->status = BACKUP_STATUS_ORPHAN; + else if (strcmp(status, "CORRUPT") == 0) + backup->status = BACKUP_STATUS_CORRUPT; + else + elog(WARNING, "Invalid STATUS \"%s\"", status); + free(status); + } + + if (parent_backup) + { + backup->parent_backup = base36dec(parent_backup); + free(parent_backup); + } + + if (program_version) + { + StrNCpy(backup->program_version, program_version, + sizeof(backup->program_version)); + pfree(program_version); + } + + if (server_version) + { + StrNCpy(backup->server_version, server_version, + sizeof(backup->server_version)); + pfree(server_version); + } + + if (compress_alg) + backup->compress_alg = parse_compress_alg(compress_alg); + + return backup; +} + +BackupMode +parse_backup_mode(const char *value) +{ + const char *v = value; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0 && pg_strncasecmp("full", v, len) == 0) + return BACKUP_MODE_FULL; + else if (len > 0 && pg_strncasecmp("page", v, len) == 0) + return BACKUP_MODE_DIFF_PAGE; + else if (len > 0 && pg_strncasecmp("ptrack", v, len) == 0) + return BACKUP_MODE_DIFF_PTRACK; + else if (len > 0 && pg_strncasecmp("delta", v, len) == 0) + return BACKUP_MODE_DIFF_DELTA; + + /* Backup mode is invalid, so leave with an error */ + elog(ERROR, "invalid backup-mode \"%s\"", value); + return BACKUP_MODE_INVALID; +} + +const char * +deparse_backup_mode(BackupMode mode) +{ + switch (mode) + { + case BACKUP_MODE_FULL: + return "full"; + case BACKUP_MODE_DIFF_PAGE: + return "page"; + case BACKUP_MODE_DIFF_PTRACK: + return "ptrack"; + case BACKUP_MODE_DIFF_DELTA: + return "delta"; + case BACKUP_MODE_INVALID: + return "invalid"; + } + + return NULL; +} + +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + +/* + * Fill pgBackup struct with default values. + */ +void +pgBackupInit(pgBackup *backup) +{ + backup->backup_id = INVALID_BACKUP_ID; + backup->backup_mode = BACKUP_MODE_INVALID; + backup->status = BACKUP_STATUS_INVALID; + backup->tli = 0; + backup->start_lsn = 0; + backup->stop_lsn = 0; + backup->start_time = (time_t) 0; + backup->end_time = (time_t) 0; + backup->recovery_xid = 0; + backup->recovery_time = (time_t) 0; + + backup->data_bytes = BYTES_INVALID; + backup->wal_bytes = BYTES_INVALID; + + backup->compress_alg = COMPRESS_ALG_DEFAULT; + backup->compress_level = COMPRESS_LEVEL_DEFAULT; + + backup->block_size = BLCKSZ; + backup->wal_block_size = XLOG_BLCKSZ; + backup->checksum_version = 0; + + backup->stream = false; + backup->from_replica = false; + backup->parent_backup = INVALID_BACKUP_ID; + backup->parent_backup_link = NULL; + backup->primary_conninfo = NULL; + backup->program_version[0] = '\0'; + backup->server_version[0] = '\0'; +} + +/* + * Copy backup metadata from **src** into **dst**. + */ +void +pgBackupCopy(pgBackup *dst, pgBackup *src) +{ + pfree(dst->primary_conninfo); + + memcpy(dst, src, sizeof(pgBackup)); + + if (src->primary_conninfo) + dst->primary_conninfo = pstrdup(src->primary_conninfo); +} + +/* free pgBackup object */ +void +pgBackupFree(void *backup) +{ + pgBackup *b = (pgBackup *) backup; + + pfree(b->primary_conninfo); + pfree(backup); +} + +/* Compare two pgBackup with their IDs (start time) in ascending order */ +int +pgBackupCompareId(const void *l, const void *r) +{ + pgBackup *lp = *(pgBackup **)l; + pgBackup *rp = *(pgBackup **)r; + + if (lp->start_time > rp->start_time) + return 1; + else if (lp->start_time < rp->start_time) + return -1; + else + return 0; +} + +/* Compare two pgBackup with their IDs in descending order */ +int +pgBackupCompareIdDesc(const void *l, const void *r) +{ + return -pgBackupCompareId(l, r); +} + +/* + * Construct absolute path of the backup directory. + * If subdir is not NULL, it will be appended after the path. + */ +void +pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) +{ + pgBackupGetPath2(backup, path, len, subdir, NULL); +} + +/* + * Construct absolute path of the backup directory. + * Append "subdir1" and "subdir2" to the backup directory. + */ +void +pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2) +{ + /* If "subdir1" is NULL do not check "subdir2" */ + if (!subdir1) + snprintf(path, len, "%s/%s", backup_instance_path, + base36enc(backup->start_time)); + else if (!subdir2) + snprintf(path, len, "%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1); + /* "subdir1" and "subdir2" is not NULL */ + else + snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1, subdir2); + + make_native_path(path); +} + +/* Find parent base FULL backup for current backup using parent_backup_link, + * return NULL if not found + */ +pgBackup* +find_parent_backup(pgBackup *current_backup) +{ + pgBackup *base_full_backup = NULL; + base_full_backup = current_backup; + + while (base_full_backup->backup_mode != BACKUP_MODE_FULL) + { + /* + * If we haven't found parent for incremental backup, + * mark it and all depending backups as orphaned + */ + if (base_full_backup->parent_backup_link == NULL + || (base_full_backup->status != BACKUP_STATUS_OK + && base_full_backup->status != BACKUP_STATUS_DONE)) + { + pgBackup *orphaned_backup = current_backup; + + while (orphaned_backup != NULL) + { + orphaned_backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(orphaned_backup); + if (base_full_backup->parent_backup_link == NULL) + elog(WARNING, "Backup %s is orphaned because its parent backup is not found", + base36enc(orphaned_backup->start_time)); + else + elog(WARNING, "Backup %s is orphaned because its parent backup is corrupted", + base36enc(orphaned_backup->start_time)); + + orphaned_backup = orphaned_backup->parent_backup_link; + } + + base_full_backup = NULL; + break; + } + + base_full_backup = base_full_backup->parent_backup_link; + } + + return base_full_backup; +} diff --git a/src/configure.c b/src/configure.c new file mode 100644 index 00000000..8b86e438 --- /dev/null +++ b/src/configure.c @@ -0,0 +1,490 @@ +/*------------------------------------------------------------------------- + * + * configure.c: - manage backup catalog. + * + * Copyright (c) 2017-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "utils/logger.h" + +#include "pqexpbuffer.h" + +#include "utils/json.h" + + +static void opt_log_level_console(pgut_option *opt, const char *arg); +static void opt_log_level_file(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); + +static void show_configure_start(void); +static void show_configure_end(void); +static void show_configure(pgBackupConfig *config); + +static void show_configure_json(pgBackupConfig *config); + +static pgBackupConfig *cur_config = NULL; + +static PQExpBufferData show_buf; +static int32 json_level = 0; + +/* + * All this code needs refactoring. + */ + +/* Set configure options */ +int +do_configure(bool show_only) +{ + pgBackupConfig *config = readBackupCatalogConfigFile(); + if (pgdata) + config->pgdata = pgdata; + if (pgut_dbname) + config->pgdatabase = pgut_dbname; + if (host) + config->pghost = host; + if (port) + config->pgport = port; + if (username) + config->pguser = username; + + if (master_host) + config->master_host = master_host; + if (master_port) + config->master_port = master_port; + if (master_db) + config->master_db = master_db; + if (master_user) + config->master_user = master_user; + + if (replica_timeout) + config->replica_timeout = replica_timeout; + + if (archive_timeout) + config->archive_timeout = archive_timeout; + + if (log_level_console) + config->log_level_console = log_level_console; + if (log_level_file) + config->log_level_file = log_level_file; + if (log_filename) + config->log_filename = log_filename; + if (error_log_filename) + config->error_log_filename = error_log_filename; + if (log_directory) + config->log_directory = log_directory; + if (log_rotation_size) + config->log_rotation_size = log_rotation_size; + if (log_rotation_age) + config->log_rotation_age = log_rotation_age; + + if (retention_redundancy) + config->retention_redundancy = retention_redundancy; + if (retention_window) + config->retention_window = retention_window; + + if (compress_alg) + config->compress_alg = compress_alg; + if (compress_level) + config->compress_level = compress_level; + + if (show_only) + show_configure(config); + else + writeBackupCatalogConfigFile(config); + + return 0; +} + +void +pgBackupConfigInit(pgBackupConfig *config) +{ + config->system_identifier = 0; + +#if PG_VERSION_NUM >= 110000 + config->xlog_seg_size = 0; +#else + config->xlog_seg_size = XLOG_SEG_SIZE; +#endif + + config->pgdata = NULL; + config->pgdatabase = NULL; + config->pghost = NULL; + config->pgport = NULL; + config->pguser = NULL; + + config->master_host = NULL; + config->master_port = NULL; + config->master_db = NULL; + config->master_user = NULL; + config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; + + config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; + + config->log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; + config->log_level_file = LOG_LEVEL_FILE_DEFAULT; + config->log_filename = LOG_FILENAME_DEFAULT; + config->error_log_filename = NULL; + config->log_directory = LOG_DIRECTORY_DEFAULT; + config->log_rotation_size = LOG_ROTATION_SIZE_DEFAULT; + config->log_rotation_age = LOG_ROTATION_AGE_DEFAULT; + + config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; + config->retention_window = RETENTION_WINDOW_DEFAULT; + + config->compress_alg = COMPRESS_ALG_DEFAULT; + config->compress_level = COMPRESS_LEVEL_DEFAULT; +} + +void +writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) +{ + uint64 res; + const char *unit; + + fprintf(out, "#Backup instance info\n"); + fprintf(out, "PGDATA = %s\n", config->pgdata); + fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); +#if PG_VERSION_NUM >= 110000 + fprintf(out, "xlog-seg-size = %u\n", config->xlog_seg_size); +#endif + + fprintf(out, "#Connection parameters:\n"); + if (config->pgdatabase) + fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); + if (config->pghost) + fprintf(out, "PGHOST = %s\n", config->pghost); + if (config->pgport) + fprintf(out, "PGPORT = %s\n", config->pgport); + if (config->pguser) + fprintf(out, "PGUSER = %s\n", config->pguser); + + fprintf(out, "#Replica parameters:\n"); + if (config->master_host) + fprintf(out, "master-host = %s\n", config->master_host); + if (config->master_port) + fprintf(out, "master-port = %s\n", config->master_port); + if (config->master_db) + fprintf(out, "master-db = %s\n", config->master_db); + if (config->master_user) + fprintf(out, "master-user = %s\n", config->master_user); + + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Archive parameters:\n"); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "archive-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Logging parameters:\n"); + fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); + fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); + fprintf(out, "log-filename = %s\n", config->log_filename); + if (config->error_log_filename) + fprintf(out, "error-log-filename = %s\n", config->error_log_filename); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + fprintf(out, "log-directory = %s/%s\n", backup_path, config->log_directory); + else + fprintf(out, "log-directory = %s\n", config->log_directory); + /* Convert values from base unit */ + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, (res)?unit:"KB"); + + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, (res)?unit:"min"); + + fprintf(out, "#Retention parameters:\n"); + fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); + fprintf(out, "retention-window = %u\n", config->retention_window); + + fprintf(out, "#Compression parameters:\n"); + + fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); + fprintf(out, "compress-level = %d\n", config->compress_level); +} + +void +writeBackupCatalogConfigFile(pgBackupConfig *config) +{ + char path[MAXPGPATH]; + FILE *fp; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot create %s: %s", + BACKUP_CATALOG_CONF_FILE, strerror(errno)); + + writeBackupCatalogConfig(fp, config); + fclose(fp); +} + + +pgBackupConfig* +readBackupCatalogConfigFile(void) +{ + pgBackupConfig *config = pgut_new(pgBackupConfig); + char path[MAXPGPATH]; + + pgut_option options[] = + { + /* retention options */ + { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, + { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, + /* compression options */ + { 'f', 0, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 0, "compress-level", &(config->compress_level), SOURCE_CMDLINE }, + /* logging options */ + { 'f', 0, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, + { 'f', 0, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, + { 's', 0, "log-filename", &(config->log_filename), SOURCE_CMDLINE }, + { 's', 0, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, + { 's', 0, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, + { 'u', 0, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, + { 'u', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* connection options */ + { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, + { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, + { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, + { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, + { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, + /* replica options */ + { 's', 0, "master-host", &(config->master_host), SOURCE_FILE_STRICT }, + { 's', 0, "master-port", &(config->master_port), SOURCE_FILE_STRICT }, + { 's', 0, "master-db", &(config->master_db), SOURCE_FILE_STRICT }, + { 's', 0, "master-user", &(config->master_user), SOURCE_FILE_STRICT }, + { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* other options */ + { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, +#if PG_VERSION_NUM >= 110000 + {'u', 0, "xlog-seg-size", &config->xlog_seg_size, SOURCE_FILE_STRICT}, +#endif + /* archive options */ + { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + {0} + }; + + cur_config = config; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + + pgBackupConfigInit(config); + pgut_readopt(path, options, ERROR, true); + +#if PG_VERSION_NUM >= 110000 + if (!IsValidWalSegSize(config->xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", config->xlog_seg_size); +#endif + + return config; +} + +/* + * Read xlog-seg-size from BACKUP_CATALOG_CONF_FILE. + */ +uint32 +get_config_xlog_seg_size(void) +{ +#if PG_VERSION_NUM >= 110000 + char path[MAXPGPATH]; + uint32 seg_size; + pgut_option options[] = + { + {'u', 0, "xlog-seg-size", &seg_size, SOURCE_FILE_STRICT}, + {0} + }; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, false); + + if (!IsValidWalSegSize(seg_size)) + elog(ERROR, "Invalid WAL segment size %u", seg_size); + + return seg_size; + +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + +static void +opt_log_level_console(pgut_option *opt, const char *arg) +{ + cur_config->log_level_console = parse_log_level(arg); +} + +static void +opt_log_level_file(pgut_option *opt, const char *arg) +{ + cur_config->log_level_file = parse_log_level(arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + cur_config->compress_alg = parse_compress_alg(arg); +} + +/* + * Initialize configure visualization. + */ +static void +show_configure_start(void) +{ + if (show_format == SHOW_PLAIN) + return; + + /* For now we need buffer only for JSON format */ + json_level = 0; + initPQExpBuffer(&show_buf); +} + +/* + * Finalize configure visualization. + */ +static void +show_configure_end(void) +{ + if (show_format == SHOW_PLAIN) + return; + else + appendPQExpBufferChar(&show_buf, '\n'); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show configure information of pg_probackup. + */ +static void +show_configure(pgBackupConfig *config) +{ + show_configure_start(); + + if (show_format == SHOW_PLAIN) + writeBackupCatalogConfig(stdout, config); + else + show_configure_json(config); + + show_configure_end(); +} + +/* + * Json output. + */ + +static void +show_configure_json(pgBackupConfig *config) +{ + PQExpBuffer buf = &show_buf; + uint64 res; + const char *unit; + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "pgdata", config->pgdata, json_level, false); + + json_add_key(buf, "system-identifier", json_level, true); + appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); + +#if PG_VERSION_NUM >= 110000 + json_add_key(buf, "xlog-seg-size", json_level, true); + appendPQExpBuffer(buf, "%u", config->xlog_seg_size); +#endif + + /* Connection parameters */ + if (config->pgdatabase) + json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); + if (config->pghost) + json_add_value(buf, "pghost", config->pghost, json_level, true); + if (config->pgport) + json_add_value(buf, "pgport", config->pgport, json_level, true); + if (config->pguser) + json_add_value(buf, "pguser", config->pguser, json_level, true); + + /* Replica parameters */ + if (config->master_host) + json_add_value(buf, "master-host", config->master_host, json_level, + true); + if (config->master_port) + json_add_value(buf, "master-port", config->master_port, json_level, + true); + if (config->master_db) + json_add_value(buf, "master-db", config->master_db, json_level, true); + if (config->master_user) + json_add_value(buf, "master-user", config->master_user, json_level, + true); + + json_add_key(buf, "replica-timeout", json_level, true); + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Archive parameters */ + json_add_key(buf, "archive-timeout", json_level, true); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Logging parameters */ + json_add_value(buf, "log-level-console", + deparse_log_level(config->log_level_console), json_level, + true); + json_add_value(buf, "log-level-file", + deparse_log_level(config->log_level_file), json_level, + true); + json_add_value(buf, "log-filename", config->log_filename, json_level, + true); + if (config->error_log_filename) + json_add_value(buf, "error-log-filename", config->error_log_filename, + json_level, true); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + { + char log_directory_fullpath[MAXPGPATH]; + + sprintf(log_directory_fullpath, "%s/%s", + backup_path, config->log_directory); + + json_add_value(buf, "log-directory", log_directory_fullpath, + json_level, true); + } + else + json_add_value(buf, "log-directory", config->log_directory, + json_level, true); + + json_add_key(buf, "log-rotation-size", json_level, true); + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"KB"); + + json_add_key(buf, "log-rotation-age", json_level, true); + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"min"); + + /* Retention parameters */ + json_add_key(buf, "retention-redundancy", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_redundancy); + + json_add_key(buf, "retention-window", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_window); + + /* Compression parameters */ + json_add_value(buf, "compress-algorithm", + deparse_compress_alg(config->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", config->compress_level); + + json_add(buf, JT_END_OBJECT, &json_level); +} diff --git a/src/data.c b/src/data.c new file mode 100644 index 00000000..a66770bc --- /dev/null +++ b/src/data.c @@ -0,0 +1,1407 @@ +/*------------------------------------------------------------------------- + * + * data.c: utils to parse and backup data pages + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "libpq/pqsignal.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum_impl.h" +#include + +#ifdef HAVE_LIBZ +#include +#endif + +#ifdef HAVE_LIBZ +/* Implementation of zlib compression method */ +static int32 +zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, + int level) +{ + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, + level); + + return rc == Z_OK ? compressed_size : rc; +} + +/* Implementation of zlib compression method */ +static int32 +zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) +{ + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); + + return rc == Z_OK ? dest_len : rc; +} +#endif + +/* + * Compresses source into dest using algorithm. Returns the number of bytes + * written in the destination buffer, or -1 if compression fails. + */ +static int32 +do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, int level) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; +#ifdef HAVE_LIBZ + case ZLIB_COMPRESS: + return zlib_compress(dst, dst_size, src, src_size, level); +#endif + case PGLZ_COMPRESS: + return pglz_compress(src, src_size, dst, PGLZ_strategy_always); + } + + return -1; +} + +/* + * Decompresses source into dest using algorithm. Returns the number of bytes + * decompressed in the destination buffer, or -1 if decompression fails. + */ +static int32 +do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; +#ifdef HAVE_LIBZ + case ZLIB_COMPRESS: + return zlib_decompress(dst, dst_size, src, src_size); +#endif + case PGLZ_COMPRESS: + return pglz_decompress(src, src_size, dst, dst_size); + } + + return -1; +} + +/* + * When copying datafiles to backup we validate and compress them block + * by block. Thus special header is required for each data block. + */ +typedef struct BackupPageHeader +{ + BlockNumber block; /* block number */ + int32 compressed_size; +} BackupPageHeader; + +/* Special value for compressed_size field */ +#define PageIsTruncated -2 +#define SkipCurrentPage -3 + +/* Verify page's header */ +static bool +parse_page(Page page, XLogRecPtr *lsn) +{ + PageHeader phdr = (PageHeader) page; + + /* Get lsn from page header */ + *lsn = PageXLogRecPtrGet(phdr->pd_lsn); + + if (PageGetPageSize(phdr) == BLCKSZ && + PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && + phdr->pd_lower >= SizeOfPageHeaderData && + phdr->pd_lower <= phdr->pd_upper && + phdr->pd_upper <= phdr->pd_special && + phdr->pd_special <= BLCKSZ && + phdr->pd_special == MAXALIGN(phdr->pd_special)) + return true; + + return false; +} + +/* Read one page from file directly accessing disk + * return value: + * 0 - if the page is not found + * 1 - if the page is found and valid + * -1 - if the page is found but invalid + */ +static int +read_page_from_file(pgFile *file, BlockNumber blknum, + FILE *in, Page page, XLogRecPtr *page_lsn) +{ + off_t offset = blknum * BLCKSZ; + size_t read_len = 0; + + /* read the block */ + if (fseek(in, offset, SEEK_SET) != 0) + elog(ERROR, "File: %s, could not seek to block %u: %s", + file->path, blknum, strerror(errno)); + + read_len = fread(page, 1, BLCKSZ, in); + + if (read_len != BLCKSZ) + { + /* The block could have been truncated. It is fine. */ + if (read_len == 0) + { + elog(LOG, "File %s, block %u, file was truncated", + file->path, blknum); + return 0; + } + else + elog(WARNING, "File: %s, block %u, expected block size %d," + "but read %lu, try again", + file->path, blknum, BLCKSZ, read_len); + } + + /* + * If we found page with invalid header, at first check if it is zeroed, + * which is a valid state for page. If it is not, read it and check header + * again, because it's possible that we've read a partly flushed page. + * If after several attempts page header is still invalid, throw an error. + * The same idea is applied to checksum verification. + */ + if (!parse_page(page, page_lsn)) + { + int i; + /* Check if the page is zeroed. */ + for(i = 0; i < BLCKSZ && page[i] == 0; i++); + + /* Page is zeroed. No need to check header and checksum. */ + if (i == BLCKSZ) + { + elog(LOG, "File: %s blknum %u, empty page", file->path, blknum); + return 1; + } + + /* + * If page is not completely empty and we couldn't parse it, + * try again several times. If it didn't help, throw error + */ + elog(LOG, "File: %s blknum %u have wrong page header, try again", + file->path, blknum); + return -1; + } + + /* Verify checksum */ + if(current.checksum_version) + { + /* + * If checksum is wrong, sleep a bit and then try again + * several times. If it didn't help, throw error + */ + if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) + != ((PageHeader) page)->pd_checksum) + { + elog(WARNING, "File: %s blknum %u have wrong checksum, try again", + file->path, blknum); + return -1; + } + else + { + /* page header and checksum are correct */ + return 1; + } + } + else + { + /* page header is correct and checksum check is disabled */ + return 1; + } +} + +/* + * Retrieves a page taking the backup mode into account + * and writes it into argument "page". Argument "page" + * should be a pointer to allocated BLCKSZ of bytes. + * + * Prints appropriate warnings/errors/etc into log. + * Returns 0 if page was successfully retrieved + * SkipCurrentPage(-3) if we need to skip this page + * PageIsTruncated(-2) if the page was truncated + */ +static int32 +prepare_page(backup_files_arg *arguments, + pgFile *file, XLogRecPtr prev_backup_start_lsn, + BlockNumber blknum, BlockNumber nblocks, + FILE *in, int *n_skipped, + BackupMode backup_mode, + Page page) +{ + XLogRecPtr page_lsn = 0; + int try_again = 100; + bool page_is_valid = false; + bool page_is_truncated = false; + BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + + /* + * Read the page and verify its header and checksum. + * Under high write load it's possible that we've read partly + * flushed page, so try several times before throwing an error. + */ + if (backup_mode != BACKUP_MODE_DIFF_PTRACK) + { + while(!page_is_valid && try_again) + { + int result = read_page_from_file(file, blknum, + in, page, &page_lsn); + + try_again--; + if (result == 0) + { + /* This block was truncated.*/ + page_is_truncated = true; + /* Page is not actually valid, but it is absent + * and we're not going to reread it or validate */ + page_is_valid = true; + } + + if (result == 1) + page_is_valid = true; + + /* + * If ptrack support is available use it to get invalid block + * instead of rereading it 99 times + */ + //elog(WARNING, "Checksum_Version: %i", current.checksum_version ? 1 : 0); + + if (result == -1 && is_ptrack_support) + { + elog(WARNING, "File %s, block %u, try to fetch via SQL", + file->path, blknum); + break; + } + } + /* + * If page is not valid after 100 attempts to read it + * throw an error. + */ + if(!page_is_valid && !is_ptrack_support) + elog(ERROR, "Data file checksum mismatch. Canceling backup"); + } + + if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) + { + size_t page_size = 0; + Page ptrack_page = NULL; + ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, + file->relOid, absolute_blknum, &page_size); + + if (ptrack_page == NULL) + { + /* This block was truncated.*/ + page_is_truncated = true; + } + else if (page_size != BLCKSZ) + { + free(ptrack_page); + elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", + file->path, absolute_blknum, BLCKSZ, page_size); + } + else + { + /* + * We need to copy the page that was successfully + * retreieved from ptrack into our output "page" parameter. + * We must set checksum here, because it is outdated + * in the block recieved from shared buffers. + */ + memcpy(page, ptrack_page, BLCKSZ); + free(ptrack_page); + if (is_checksum_enabled) + ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); + } + /* get lsn from page, provided by pg_ptrack_get_block() */ + if (backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev && + !page_is_truncated && + !parse_page(page, &page_lsn)) + elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " + "Possible risk of a memory corruption"); + + } + + if (backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev && + !page_is_truncated && + page_lsn < prev_backup_start_lsn) + { + elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path); + (*n_skipped)++; + return SkipCurrentPage; + } + + if (page_is_truncated) + return PageIsTruncated; + + return 0; +} + +static void +compress_and_backup_page(pgFile *file, BlockNumber blknum, + FILE *in, FILE *out, pg_crc32 *crc, + int page_state, Page page, + CompressAlg calg, int clevel) +{ + BackupPageHeader header; + size_t write_buffer_size = sizeof(header); + char write_buffer[BLCKSZ+sizeof(header)]; + char compressed_page[BLCKSZ]; + + if(page_state == SkipCurrentPage) + return; + + header.block = blknum; + header.compressed_size = page_state; + + if(page_state == PageIsTruncated) + { + /* + * The page was truncated. Write only header + * to know that we must truncate restored file + */ + memcpy(write_buffer, &header, sizeof(header)); + } + else + { + /* The page was not truncated, so we need to compress it */ + header.compressed_size = do_compress(compressed_page, BLCKSZ, + page, BLCKSZ, calg, clevel); + + file->compress_alg = calg; + file->read_size += BLCKSZ; + Assert (header.compressed_size <= BLCKSZ); + + /* The page was successfully compressed. */ + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), + compressed_page, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + /* Nonpositive value means that compression failed. Write it as is. */ + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page, BLCKSZ); + write_buffer_size += header.compressed_size; + } + } + + /* elog(VERBOSE, "backup blkno %u, compressed_size %d write_buffer_size %ld", + blknum, header.compressed_size, write_buffer_size); */ + + /* Update CRC */ + COMP_CRC32C(*crc, write_buffer, write_buffer_size); + + /* write data page */ + if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) + { + int errno_tmp = errno; + + fclose(in); + fclose(out); + elog(ERROR, "File: %s, cannot write backup at block %u : %s", + file->path, blknum, strerror(errno_tmp)); + } + + file->write_size += write_buffer_size; +} + +/* + * Backup data file in the from_root directory to the to_root directory with + * same relative path. If prev_backup_start_lsn is not NULL, only pages with + * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental backup), validate checksum, optionally compress and write to + * backup with special header. + */ +bool +backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel) +{ + FILE *in; + FILE *out; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int n_blocks_read = 0; + int page_state; + char curr_page[BLCKSZ]; + + /* + * Skip unchanged file only if it exists in previous backup. + * This way we can correctly handle null-sized files which are + * not tracked by pagemap and thus always marked as unchanged. + */ + if ((backup_mode == BACKUP_MODE_DIFF_PAGE || + backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->pagemap.bitmapsize == PageBitmapIsEmpty && + file->exists_in_prev && !file->pagemap_isabsent) + { + /* + * There are no changed blocks since last backup. We want make + * incremental backup, so we should exit. + */ + elog(VERBOSE, "Skipping the unchanged file: %s", file->path); + return false; + } + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + INIT_CRC32C(file->crc); + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(file->crc); + + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + { + elog(LOG, "File \"%s\" is not found", file->path); + return false; + } + + elog(ERROR, "cannot open file \"%s\": %s", + file->path, strerror(errno)); + } + + if (file->size % BLCKSZ != 0) + { + fclose(in); + elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size); + } + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + nblocks = file->size/BLCKSZ; + + /* open backup file for write */ + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open backup file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* + * Read each page, verify checksum and write it to backup. + * If page map is empty or file is not present in previous backup + * backup all pages of the relation. + * + * We will enter here if backup_mode is FULL or DELTA. + */ + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev) + { + for (blknum = 0; blknum < nblocks; blknum++) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } + if (backup_mode == BACKUP_MODE_DIFF_DELTA) + file->n_blocks = n_blocks_read; + } + /* + * If page map is not empty we scan only changed blocks. + * + * We will enter here if backup_mode is PAGE or PTRACK. + */ + else + { + datapagemap_iterator_t *iter; + iter = datapagemap_iterate(&file->pagemap); + while (datapagemap_next(iter, &blknum)) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } + + pg_free(file->pagemap.bitmap); + pg_free(iter); + } + + /* update file permission */ + if (chmod(to_path, FILE_PERMISSION) == -1) + { + int errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", file->path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write backup file \"%s\": %s", + to_path, strerror(errno)); + fclose(in); + + FIN_CRC32C(file->crc); + + /* + * If we have pagemap then file in the backup can't be a zero size. + * Otherwise, we will clear the last file. + */ + if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) + { + if (remove(to_path) == -1) + elog(ERROR, "cannot remove file \"%s\": %s", to_path, + strerror(errno)); + return false; + } + + return true; +} + +/* + * Restore files in the from_root directory to the to_root directory with + * same relative path. + * + * If write_header is true then we add header to each restored block, currently + * it is used for MERGE command. + */ +void +restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, + bool write_header) +{ + FILE *in = NULL; + FILE *out = NULL; + BackupPageHeader header; + BlockNumber blknum = 0, + truncate_from = 0; + bool need_truncate = false; + + /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ + if (file->write_size != BYTES_INVALID) + { + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + elog(ERROR, "cannot open backup file \"%s\": %s", file->path, + strerror(errno)); + } + } + + /* + * Open backup file for write. We use "r+" at first to overwrite only + * modified pages for differential restore. If the file does not exist, + * re-open it with "w" to create an empty file. + */ + out = fopen(to_path, PG_BINARY_R "+"); + if (out == NULL && errno == ENOENT) + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open restore target file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + while (true) + { + off_t write_pos; + size_t read_len; + DataPage compressed_page; /* used as read buffer */ + DataPage page; + + /* File didn`t changed. Nothig to copy */ + if (file->write_size == BYTES_INVALID) + break; + + /* + * We need to truncate result file if data file in a incremental backup + * less than data file in a full backup. We know it thanks to n_blocks. + * + * It may be equal to -1, then we don't want to truncate the result + * file. + */ + if (file->n_blocks != BLOCKNUM_INVALID && + (blknum + 1) > file->n_blocks) + { + truncate_from = blknum; + need_truncate = true; + break; + } + + /* read BackupPageHeader */ + read_len = fread(&header, 1, sizeof(header), in); + if (read_len != sizeof(header)) + { + int errno_tmp = errno; + if (read_len == 0 && feof(in)) + break; /* EOF found */ + else if (read_len != 0 && feof(in)) + elog(ERROR, + "odd size page found at block %u of \"%s\"", + blknum, file->path); + else + elog(ERROR, "cannot read header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno_tmp)); + } + + if (header.block < blknum) + elog(ERROR, "backup is broken at file->path %s block %u", + file->path, blknum); + + blknum = header.block; + + if (header.compressed_size == PageIsTruncated) + { + /* + * Backup contains information that this block was truncated. + * We need to truncate file to this length. + */ + truncate_from = blknum; + need_truncate = true; + break; + } + + Assert(header.compressed_size <= BLCKSZ); + + read_len = fread(compressed_page.data, 1, + MAXALIGN(header.compressed_size), in); + if (read_len != MAXALIGN(header.compressed_size)) + elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + blknum, file->path, read_len, header.compressed_size); + + if (header.compressed_size != BLCKSZ) + { + int32 uncompressed_size = 0; + + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + MAXALIGN(header.compressed_size), + file->compress_alg); + + if (uncompressed_size != BLCKSZ) + elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + file->path, uncompressed_size); + } + + write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : + blknum * BLCKSZ; + + /* + * Seek and write the restored page. + */ + if (fseek(out, write_pos, SEEK_SET) < 0) + elog(ERROR, "cannot seek block %u of \"%s\": %s", + blknum, to_path, strerror(errno)); + + if (write_header) + { + if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) + elog(ERROR, "cannot write header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + + if (header.compressed_size < BLCKSZ) + { + if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + else + { + /* if page wasn't compressed, we've read full block */ + if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + } + + /* + * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do + * But during DELTA backup we read every file in PGDATA and thus DELTA backup + * knows exact size of every file at the time of backup. + * So when restoring file from DELTA backup we, knowning it`s size at + * a time of a backup, can truncate file to this size. + */ + if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) + { + size_t file_size = 0; + + /* get file current size */ + fseek(out, 0, SEEK_END); + file_size = ftell(out); + + if (file_size > file->n_blocks * BLCKSZ) + { + truncate_from = file->n_blocks; + need_truncate = true; + } + } + + if (need_truncate) + { + off_t write_pos; + + write_pos = (write_header) ? truncate_from * (BLCKSZ + sizeof(header)) : + truncate_from * BLCKSZ; + + /* + * Truncate file to this length. + */ + if (ftruncate(fileno(out), write_pos) != 0) + elog(ERROR, "cannot truncate \"%s\": %s", + file->path, strerror(errno)); + elog(INFO, "Delta truncate file %s to block %u", + file->path, truncate_from); + } + + /* update file permission */ + if (chmod(to_path, file->mode) == -1) + { + int errno_tmp = errno; + + if (in) + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + if (in) + fclose(in); +} + +/* + * Copy file to backup. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +bool +copy_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + struct stat st; + pg_crc32 crc; + + INIT_CRC32C(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(crc); + file->crc = crc; + + /* maybe deleted, it's not error */ + if (errno == ENOENT) + return false; + + elog(ERROR, "cannot open source file \"%s\": %s", file->path, + strerror(errno)); + } + + /* open backup file for write */ + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot stat \"%s\": %s", file->path, + strerror(errno)); + } + + /* copy content and calc CRC */ + for (;;) + { + read_len = 0; + + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + break; + + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->read_size += read_len; + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* copy odd part. */ + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->read_size += read_len; + } + + file->write_size = (int64) file->read_size; + /* finish CRC calculation and store into pgFile */ + FIN_CRC32C(crc); + file->crc = crc; + + /* update file permission */ + if (chmod(to_path, st.st_mode) == -1) + { + errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + fclose(in); + + return true; +} + +/* + * Move file from one backup to another. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +void +move_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + if (rename(file->path, to_path) == -1) + elog(ERROR, "Cannot move file \"%s\" to path \"%s\": %s", + file->path, to_path, strerror(errno)); +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf, int errnum) +{ + int gz_errnum; + const char *errmsg; + + errmsg = gzerror(gzf, &gz_errnum); + if (gz_errnum == Z_ERRNO) + return strerror(errnum); + else + return errmsg; +} +#endif + +/* + * Copy file attributes + */ +static void +copy_meta(const char *from_path, const char *to_path, bool unlink_on_error) +{ + struct stat st; + + if (stat(from_path, &st) == -1) + { + if (unlink_on_error) + unlink(to_path); + elog(ERROR, "Cannot stat file \"%s\": %s", + from_path, strerror(errno)); + } + + if (chmod(to_path, st.st_mode) == -1) + { + if (unlink_on_error) + unlink(to_path); + elog(ERROR, "Cannot change mode of file \"%s\": %s", + to_path, strerror(errno)); + } +} + +/* + * Copy WAL segment from pgdata to archive catalog with possible compression. + */ +void +push_wal_file(const char *from_path, const char *to_path, bool is_compress, + bool overwrite) +{ + FILE *in = NULL; + FILE *out=NULL; + char buf[XLOG_BLCKSZ]; + const char *to_path_p = to_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + +#ifdef HAVE_LIBZ + char gz_to_path[MAXPGPATH]; + gzFile gz_out = NULL; +#endif + + /* open file for read */ + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, + strerror(errno)); + + /* open backup file for write */ +#ifdef HAVE_LIBZ + if (is_compress) + { + snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); + + if (!overwrite && fileExists(gz_to_path)) + elog(ERROR, "WAL segment \"%s\" already exists.", gz_to_path); + + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); + + gz_out = gzopen(to_path_temp, PG_BINARY_W); + if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) + elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", + compress_level, to_path_temp, get_gz_error(gz_out, errno)); + + to_path_p = gz_to_path; + } + else +#endif + { + if (!overwrite && fileExists(to_path)) + elog(ERROR, "WAL segment \"%s\" already exists.", to_path); + + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fopen(to_path_temp, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + + /* copy content */ + for (;;) + { + size_t read_len = 0; + + read_len = fread(buf, 1, sizeof(buf), in); + + if (ferror(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, + "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + if (read_len > 0) + { +#ifdef HAVE_LIBZ + if (is_compress) + { + if (gzwrite(gz_out, buf, read_len) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + } + + if (feof(in) || read_len == 0) + break; + } + +#ifdef HAVE_LIBZ + if (is_compress) + { + if (gzclose(gz_out) != 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + + if (fclose(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + /* update file permission. */ + copy_meta(from_path, to_path_temp, true); + + if (rename(to_path_temp, to_path_p) < 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path_p, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_compress) + elog(INFO, "WAL file compressed to \"%s\"", gz_to_path); +#endif +} + +/* + * Copy WAL segment from archive catalog to pgdata with possible decompression. + */ +void +get_wal_file(const char *from_path, const char *to_path) +{ + FILE *in = NULL; + FILE *out; + char buf[XLOG_BLCKSZ]; + const char *from_path_p = from_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + bool is_decompress = false; + +#ifdef HAVE_LIBZ + char gz_from_path[MAXPGPATH]; + gzFile gz_in = NULL; +#endif + + /* open file for read */ + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) + { +#ifdef HAVE_LIBZ + /* + * Maybe we need to decompress the file. Check it with .gz + * extension. + */ + snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); + gz_in = gzopen(gz_from_path, PG_BINARY_R); + if (gz_in == NULL) + { + if (errno == ENOENT) + { + /* There is no compressed file too, raise an error below */ + } + /* Cannot open compressed file for some reason */ + else + elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", + gz_from_path, strerror(errno)); + } + else + { + /* Found compressed file */ + is_decompress = true; + from_path_p = gz_from_path; + } +#endif + /* Didn't find compressed file */ + if (!is_decompress) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", + from_path, strerror(errno)); + } + + /* open backup file for write */ + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fopen(to_path_temp, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + + /* copy content */ + for (;;) + { + size_t read_len = 0; + +#ifdef HAVE_LIBZ + if (is_decompress) + { + read_len = gzread(gz_in, buf, sizeof(buf)); + if (read_len != sizeof(buf) && !gzeof(gz_in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot read compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + read_len = fread(buf, 1, sizeof(buf), in); + if (ferror(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, + strerror(errno_temp)); + } + } + + /* Check for EOF */ +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (gzeof(gz_in) || read_len == 0) + break; + } + else +#endif + { + if (feof(in) || read_len == 0) + break; + } + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (gzclose(gz_in) != 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + if (fclose(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + /* update file permission. */ + copy_meta(from_path_p, to_path_temp, true); + + if (rename(to_path_temp, to_path) < 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + elog(INFO, "WAL file decompressed from \"%s\"", gz_from_path); +#endif +} + +/* + * Calculate checksum of various files which are not copied from PGDATA, + * but created in process of backup, such as stream XLOG files, + * PG_TABLESPACE_MAP_FILE and PG_BACKUP_LABEL_FILE. + */ +bool +calc_file_checksum(pgFile *file) +{ + FILE *in; + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + struct stat st; + pg_crc32 crc; + + Assert(S_ISREG(file->mode)); + INIT_CRC32C(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(crc); + file->crc = crc; + + /* maybe deleted, it's not error */ + if (errno == ENOENT) + return false; + + elog(ERROR, "cannot open source file \"%s\": %s", file->path, + strerror(errno)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + elog(ERROR, "cannot stat \"%s\": %s", file->path, + strerror(errno)); + } + + for (;;) + { + read_len = fread(buf, 1, sizeof(buf), in); + + if(read_len == 0) + break; + + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->write_size += read_len; + file->read_size += read_len; + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* finish CRC calculation and store into pgFile */ + FIN_CRC32C(crc); + file->crc = crc; + + fclose(in); + + return true; +} diff --git a/src/delete.c b/src/delete.c new file mode 100644 index 00000000..de29d2cf --- /dev/null +++ b/src/delete.c @@ -0,0 +1,464 @@ +/*------------------------------------------------------------------------- + * + * delete.c: delete backup files. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +static int pgBackupDeleteFiles(pgBackup *backup); +static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size); + +int +do_delete(time_t backup_id) +{ + int i; + parray *backup_list, + *delete_list; + pgBackup *target_backup = NULL; + time_t parent_id = 0; + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get complete list of backups */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + if (backup_id != 0) + { + delete_list = parray_new(); + + /* Find backup to be deleted and make increment backups array to be deleted */ + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->start_time == backup_id) + { + parray_append(delete_list, backup); + + /* + * Do not remove next backups, if target backup was finished + * incorrectly. + */ + if (backup->status == BACKUP_STATUS_ERROR) + break; + + /* Save backup id to retreive increment backups */ + parent_id = backup->start_time; + target_backup = backup; + } + else if (target_backup) + { + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->parent_backup == parent_id) + { + /* Append to delete list increment backup */ + parray_append(delete_list, backup); + /* Save backup id to retreive increment backups */ + parent_id = backup->start_time; + } + else + break; + } + } + + if (parray_num(delete_list) == 0) + elog(ERROR, "no backup found, cannot delete"); + + /* Delete backups from the end of list */ + for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); + + if (interrupted) + elog(ERROR, "interrupted during delete backup"); + + pgBackupDeleteFiles(backup); + } + + parray_free(delete_list); + } + + /* Clean WAL segments */ + if (delete_wal) + { + Assert(target_backup); + + /* Find oldest LSN, used by backups */ + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->status == BACKUP_STATUS_OK) + { + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + break; + } + } + + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + } + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + return 0; +} + +/* + * Remove backups by retention policy. Retention policy is configured by + * retention_redundancy and retention_window variables. + */ +int +do_retention_purge(void) +{ + parray *backup_list; + uint32 backup_num; + size_t i; + time_t days_threshold = time(NULL) - (retention_window * 60 * 60 * 24); + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + bool keep_next_backup = true; /* Do not delete first full backup */ + bool backup_deleted = false; /* At least one backup was deleted */ + + if (delete_expired) + { + if (retention_redundancy > 0) + elog(LOG, "REDUNDANCY=%u", retention_redundancy); + if (retention_window > 0) + elog(LOG, "WINDOW=%u", retention_window); + + if (retention_redundancy == 0 + && retention_window == 0) + { + elog(WARNING, "Retention policy is not set"); + if (!delete_wal) + return 0; + } + } + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get a complete list of backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + if (parray_num(backup_list) == 0) + { + elog(INFO, "backup list is empty, purging won't be executed"); + return 0; + } + + /* Find target backups to be deleted */ + if (delete_expired && + (retention_redundancy > 0 || retention_window > 0)) + { + backup_num = 0; + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + uint32 backup_num_evaluate = backup_num; + + /* Consider only validated and correct backups */ + if (backup->status != BACKUP_STATUS_OK) + continue; + /* + * When a valid full backup was found, we can delete the + * backup that is older than it using the number of generations. + */ + if (backup->backup_mode == BACKUP_MODE_FULL) + backup_num++; + + /* Evaluate retention_redundancy if this backup is eligible for removal */ + if (keep_next_backup || + retention_redundancy >= backup_num_evaluate + 1 || + (retention_window > 0 && backup->recovery_time >= days_threshold)) + { + /* Save LSN and Timeline to remove unnecessary WAL segments */ + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + + /* Save parent backup of this incremental backup */ + if (backup->backup_mode != BACKUP_MODE_FULL) + keep_next_backup = true; + /* + * Previous incremental backup was kept or this is first backup + * so do not delete this backup. + */ + else + keep_next_backup = false; + + continue; + } + + /* Delete backup and update status to DELETED */ + pgBackupDeleteFiles(backup); + backup_deleted = true; + } + } + + /* + * If oldest_lsn and oldest_tli weren`t set because previous step was skipped + * then set them now if we are going to purge WAL + */ + if (delete_wal && (XLogRecPtrIsInvalid(oldest_lsn))) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) - 1); + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + } + + /* Be paranoid */ + if (XLogRecPtrIsInvalid(oldest_lsn)) + elog(ERROR, "Not going to purge WAL because LSN is invalid"); + + /* Purge WAL files */ + if (delete_wal) + { + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + if (backup_deleted) + elog(INFO, "Purging finished"); + else + elog(INFO, "Nothing to delete by retention policy"); + + return 0; +} + +/* + * Delete backup files of the backup and update the status of the backup to + * BACKUP_STATUS_DELETED. + */ +static int +pgBackupDeleteFiles(pgBackup *backup) +{ + size_t i; + char path[MAXPGPATH]; + char timestamp[100]; + parray *files; + + /* + * If the backup was deleted already, there is nothing to do. + */ + if (backup->status == BACKUP_STATUS_DELETED) + return 0; + + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + + elog(INFO, "delete: %s %s", + base36enc(backup->start_time), timestamp); + + /* + * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which + * the error occurs before deleting all backup files. + */ + backup->status = BACKUP_STATUS_DELETING; + pgBackupWriteBackupControlFile(backup); + + /* list files to be deleted */ + files = parray_new(); + pgBackupGetPath(backup, path, lengthof(path), NULL); + dir_list_file(files, path, false, true, true); + + /* delete leaf node first */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + /* print progress */ + elog(VERBOSE, "delete file(%zd/%lu) \"%s\"", i + 1, + (unsigned long) parray_num(files), file->path); + + if (remove(file->path)) + { + elog(WARNING, "can't remove \"%s\": %s", file->path, + strerror(errno)); + parray_walk(files, pgFileFree); + parray_free(files); + + return 1; + } + } + + parray_walk(files, pgFileFree); + parray_free(files); + backup->status = BACKUP_STATUS_DELETED; + + return 0; +} + +/* + * Deletes WAL segments up to oldest_lsn or all WAL segments (if all backups + * was deleted and so oldest_lsn is invalid). + * + * oldest_lsn - if valid, function deletes WAL segments, which contain lsn + * older than oldest_lsn. If it is invalid function deletes all WAL segments. + * oldest_tli - is used to construct oldest WAL segment in addition to + * oldest_lsn. + */ +static void +delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size) +{ + XLogSegNo targetSegNo; + char oldestSegmentNeeded[MAXFNAMELEN]; + DIR *arcdir; + struct dirent *arcde; + char wal_file[MAXPGPATH]; + char max_wal_file[MAXPGPATH]; + char min_wal_file[MAXPGPATH]; + int rc; + + max_wal_file[0] = '\0'; + min_wal_file[0] = '\0'; + + if (!XLogRecPtrIsInvalid(oldest_lsn)) + { + GetXLogSegNo(oldest_lsn, targetSegNo, xlog_seg_size); + GetXLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo, + xlog_seg_size); + + elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded); + } + else + elog(LOG, "removing all WAL segments"); + + /* + * Now it is time to do the actual work and to remove all the segments + * not needed anymore. + */ + if ((arcdir = opendir(arclog_path)) != NULL) + { + while (errno = 0, (arcde = readdir(arcdir)) != NULL) + { + /* + * We ignore the timeline part of the WAL segment identifiers in + * deciding whether a segment is still needed. This ensures that + * we won't prematurely remove a segment from a parent timeline. + * We could probably be a little more proactive about removing + * segments of non-parent timelines, but that would be a whole lot + * more complicated. + * + * We use the alphanumeric sorting property of the filenames to + * decide which ones are earlier than the exclusiveCleanupFileName + * file. Note that this means files are not removed in the order + * they were originally written, in case this worries you. + * + * We also should not forget that WAL segment can be compressed. + */ + if (IsXLogFileName(arcde->d_name) || + IsPartialXLogFileName(arcde->d_name) || + IsBackupHistoryFileName(arcde->d_name) || + IsCompressedXLogFileName(arcde->d_name)) + { + if (XLogRecPtrIsInvalid(oldest_lsn) || + strncmp(arcde->d_name + 8, oldestSegmentNeeded + 8, 16) < 0) + { + /* + * Use the original file name again now, including any + * extension that might have been chopped off before testing + * the sequence. + */ + snprintf(wal_file, MAXPGPATH, "%s/%s", + arclog_path, arcde->d_name); + + rc = unlink(wal_file); + if (rc != 0) + { + elog(WARNING, "could not remove file \"%s\": %s", + wal_file, strerror(errno)); + break; + } + elog(LOG, "removed WAL segment \"%s\"", wal_file); + + if (max_wal_file[0] == '\0' || + strcmp(max_wal_file + 8, arcde->d_name + 8) < 0) + strcpy(max_wal_file, arcde->d_name); + + if (min_wal_file[0] == '\0' || + strcmp(min_wal_file + 8, arcde->d_name + 8) > 0) + strcpy(min_wal_file, arcde->d_name); + } + } + } + + if (min_wal_file[0] != '\0') + elog(INFO, "removed min WAL segment \"%s\"", min_wal_file); + if (max_wal_file[0] != '\0') + elog(INFO, "removed max WAL segment \"%s\"", max_wal_file); + + if (errno) + elog(WARNING, "could not read archive location \"%s\": %s", + arclog_path, strerror(errno)); + if (closedir(arcdir)) + elog(WARNING, "could not close archive location \"%s\": %s", + arclog_path, strerror(errno)); + } + else + elog(WARNING, "could not open archive location \"%s\": %s", + arclog_path, strerror(errno)); +} + + +/* Delete all backup files and wal files of given instance. */ +int +do_delete_instance(void) +{ + parray *backup_list; + int i; + char instance_config_path[MAXPGPATH]; + + /* Delete all backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + pgBackupDeleteFiles(backup); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + /* Delete all wal files. */ + delete_walfiles(InvalidXLogRecPtr, 0, xlog_seg_size); + + /* Delete backup instance config file */ + join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + if (remove(instance_config_path)) + { + elog(ERROR, "can't remove \"%s\": %s", instance_config_path, + strerror(errno)); + } + + /* Delete instance root directories */ + if (rmdir(backup_instance_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + if (rmdir(arclog_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + + elog(INFO, "Instance '%s' successfully deleted", instance_name); + return 0; +} diff --git a/src/dir.c b/src/dir.c new file mode 100644 index 00000000..a08bd934 --- /dev/null +++ b/src/dir.c @@ -0,0 +1,1491 @@ +/*------------------------------------------------------------------------- + * + * dir.c: directory operation utility. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include + +#include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" +#include "datapagemap.h" + +/* + * The contents of these directories are removed or recreated during server + * start so they are not included in backups. The directories themselves are + * kept and included as empty to preserve access permissions. + */ +const char *pgdata_exclude_dir[] = +{ + PG_XLOG_DIR, + /* + * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even + * when stats_temp_directory is set because PGSS_TEXT_FILE is always created + * there. + */ + "pg_stat_tmp", + "pgsql_tmp", + + /* + * It is generally not useful to backup the contents of this directory even + * if the intention is to restore to another master. See backup.sgml for a + * more detailed description. + */ + "pg_replslot", + + /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ + "pg_dynshmem", + + /* Contents removed on startup, see AsyncShmemInit(). */ + "pg_notify", + + /* + * Old contents are loaded for possible debugging but are not required for + * normal operation, see OldSerXidInit(). + */ + "pg_serial", + + /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */ + "pg_snapshots", + + /* Contents zeroed on startup, see StartupSUBTRANS(). */ + "pg_subtrans", + + /* end of list */ + NULL, /* pg_log will be set later */ + NULL +}; + +static char *pgdata_exclude_files[] = +{ + /* Skip auto conf temporary file. */ + "postgresql.auto.conf.tmp", + + /* Skip current log file temporary file */ + "current_logfiles.tmp", + "recovery.conf", + "postmaster.pid", + "postmaster.opts", + NULL +}; + +static char *pgdata_exclude_files_non_exclusive[] = +{ + /*skip in non-exclusive backup */ + "backup_label", + "tablespace_map", + NULL +}; + +/* Tablespace mapping structures */ + +typedef struct TablespaceListCell +{ + struct TablespaceListCell *next; + char old_dir[MAXPGPATH]; + char new_dir[MAXPGPATH]; +} TablespaceListCell; + +typedef struct TablespaceList +{ + TablespaceListCell *head; + TablespaceListCell *tail; +} TablespaceList; + +typedef struct TablespaceCreatedListCell +{ + struct TablespaceCreatedListCell *next; + char link_name[MAXPGPATH]; + char linked_dir[MAXPGPATH]; +} TablespaceCreatedListCell; + +typedef struct TablespaceCreatedList +{ + TablespaceCreatedListCell *head; + TablespaceCreatedListCell *tail; +} TablespaceCreatedList; + +static int BlackListCompare(const void *str1, const void *str2); + +static bool dir_check_file(const char *root, pgFile *file); +static void dir_list_file_internal(parray *files, const char *root, + pgFile *parent, bool exclude, + bool omit_symlink, parray *black_list); + +static void list_data_directories(parray *files, const char *path, bool is_root, + bool exclude); + +/* Tablespace mapping */ +static TablespaceList tablespace_dirs = {NULL, NULL}; +static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; + +/* + * Create directory, also create parent directories if necessary. + */ +int +dir_create_dir(const char *dir, mode_t mode) +{ + char parent[MAXPGPATH]; + + strncpy(parent, dir, MAXPGPATH); + get_parent_directory(parent); + + /* Create parent first */ + if (access(parent, F_OK) == -1) + dir_create_dir(parent, mode); + + /* Create directory */ + if (mkdir(dir, mode) == -1) + { + if (errno == EEXIST) /* already exist */ + return 0; + elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); + } + + return 0; +} + +pgFile * +pgFileNew(const char *path, bool omit_symlink) +{ + struct stat st; + pgFile *file; + + /* stat the file */ + if ((omit_symlink ? stat(path, &st) : lstat(path, &st)) == -1) + { + /* file not found is not an error case */ + if (errno == ENOENT) + return NULL; + elog(ERROR, "cannot stat file \"%s\": %s", path, + strerror(errno)); + } + + file = pgFileInit(path); + file->size = st.st_size; + file->mode = st.st_mode; + + return file; +} + +pgFile * +pgFileInit(const char *path) +{ + pgFile *file; + char *file_name; + + file = (pgFile *) pgut_malloc(sizeof(pgFile)); + + file->name = NULL; + + file->size = 0; + file->mode = 0; + file->read_size = 0; + file->write_size = 0; + file->crc = 0; + file->is_datafile = false; + file->linked = NULL; + file->pagemap.bitmap = NULL; + file->pagemap.bitmapsize = PageBitmapIsEmpty; + file->pagemap_isabsent = false; + file->tblspcOid = 0; + file->dbOid = 0; + file->relOid = 0; + file->segno = 0; + file->is_database = false; + file->forkName = pgut_malloc(MAXPGPATH); + file->forkName[0] = '\0'; + + file->path = pgut_malloc(strlen(path) + 1); + strcpy(file->path, path); /* enough buffer size guaranteed */ + + /* Get file name from the path */ + file_name = strrchr(file->path, '/'); + if (file_name == NULL) + file->name = file->path; + else + { + file_name++; + file->name = file_name; + } + + file->is_cfs = false; + file->exists_in_prev = false; /* can change only in Incremental backup. */ + /* Number of blocks readed during backup */ + file->n_blocks = BLOCKNUM_INVALID; + file->compress_alg = NOT_DEFINED_COMPRESS; + return file; +} + +/* + * Delete file pointed by the pgFile. + * If the pgFile points directory, the directory must be empty. + */ +void +pgFileDelete(pgFile *file) +{ + if (S_ISDIR(file->mode)) + { + if (rmdir(file->path) == -1) + { + if (errno == ENOENT) + return; + else if (errno == ENOTDIR) /* could be symbolic link */ + goto delete_file; + + elog(ERROR, "cannot remove directory \"%s\": %s", + file->path, strerror(errno)); + } + return; + } + +delete_file: + if (remove(file->path) == -1) + { + if (errno == ENOENT) + return; + elog(ERROR, "cannot remove file \"%s\": %s", file->path, + strerror(errno)); + } +} + +pg_crc32 +pgFileGetCRC(const char *file_path) +{ + FILE *fp; + pg_crc32 crc = 0; + char buf[1024]; + size_t len; + int errno_tmp; + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", + file_path, strerror(errno)); + + /* calc CRC of backup file */ + INIT_CRC32C(crc); + while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf)) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + COMP_CRC32C(crc, buf, len); + } + errno_tmp = errno; + if (!feof(fp)) + elog(WARNING, "cannot read \"%s\": %s", file_path, + strerror(errno_tmp)); + if (len > 0) + COMP_CRC32C(crc, buf, len); + FIN_CRC32C(crc); + + fclose(fp); + + return crc; +} + +void +pgFileFree(void *file) +{ + pgFile *file_ptr; + + if (file == NULL) + return; + + file_ptr = (pgFile *) file; + + if (file_ptr->linked) + free(file_ptr->linked); + + if (file_ptr->forkName) + free(file_ptr->forkName); + + free(file_ptr->path); + free(file); +} + +/* Compare two pgFile with their path in ascending order of ASCII code. */ +int +pgFileComparePath(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->path, f2p->path); +} + +/* Compare two pgFile with their path in descending order of ASCII code. */ +int +pgFileComparePathDesc(const void *f1, const void *f2) +{ + return -pgFileComparePath(f1, f2); +} + +/* Compare two pgFile with their linked directory path. */ +int +pgFileCompareLinked(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->linked, f2p->linked); +} + +/* Compare two pgFile with their size */ +int +pgFileCompareSize(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + if (f1p->size > f2p->size) + return 1; + else if (f1p->size < f2p->size) + return -1; + else + return 0; +} + +static int +BlackListCompare(const void *str1, const void *str2) +{ + return strcmp(*(char **) str1, *(char **) str2); +} + +/* + * List files, symbolic links and directories in the directory "root" and add + * pgFile objects to "files". We add "root" to "files" if add_root is true. + * + * When omit_symlink is true, symbolic link is ignored and only file or + * directory llnked to will be listed. + */ +void +dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, + bool add_root) +{ + pgFile *file; + parray *black_list = NULL; + char path[MAXPGPATH]; + + join_path_components(path, backup_instance_path, PG_BLACK_LIST); + /* List files with black list */ + if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) + { + FILE *black_list_file = NULL; + char buf[MAXPGPATH * 2]; + char black_item[MAXPGPATH * 2]; + + black_list = parray_new(); + black_list_file = fopen(path, PG_BINARY_R); + + if (black_list_file == NULL) + elog(ERROR, "cannot open black_list: %s", strerror(errno)); + + while (fgets(buf, lengthof(buf), black_list_file) != NULL) + { + join_path_components(black_item, pgdata, buf); + + if (black_item[strlen(black_item) - 1] == '\n') + black_item[strlen(black_item) - 1] = '\0'; + + if (black_item[0] == '#' || black_item[0] == '\0') + continue; + + parray_append(black_list, black_item); + } + + fclose(black_list_file); + parray_qsort(black_list, BlackListCompare); + } + + file = pgFileNew(root, false); + if (file == NULL) + return; + + if (!S_ISDIR(file->mode)) + { + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + return; + } + if (add_root) + parray_append(files, file); + + dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); +} + +/* + * Check file or directory. + * + * Check for exclude. + * Extract information about the file parsing its name. + * Skip files: + * - skip temp tables files + * - skip unlogged tables files + * Set flags for: + * - database directories + * - datafiles + */ +static bool +dir_check_file(const char *root, pgFile *file) +{ + const char *rel_path; + int i; + int sscanf_res; + + /* Check if we need to exclude file by name */ + if (S_ISREG(file->mode)) + { + if (!exclusive_backup) + { + for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++) + if (strcmp(file->name, + pgdata_exclude_files_non_exclusive[i]) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding file: %s", file->name); + return false; + } + } + + for (i = 0; pgdata_exclude_files[i]; i++) + if (strcmp(file->name, pgdata_exclude_files[i]) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding file: %s", file->name); + return false; + } + } + /* + * If the directory name is in the exclude list, do not list the + * contents. + */ + else if (S_ISDIR(file->mode)) + { + /* + * If the item in the exclude list starts with '/', compare to + * the absolute path of the directory. Otherwise compare to the + * directory name portion. + */ + for (i = 0; pgdata_exclude_dir[i]; i++) + { + /* Full-path exclude*/ + if (pgdata_exclude_dir[i][0] == '/') + { + if (strcmp(file->path, pgdata_exclude_dir[i]) == 0) + { + elog(VERBOSE, "Excluding directory content: %s", + file->name); + return false; + } + } + else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) + { + elog(VERBOSE, "Excluding directory content: %s", + file->name); + return false; + } + } + } + + rel_path = GetRelativePath(file->path, root); + + /* + * Do not copy tablespaces twice. It may happen if the tablespace is located + * inside the PGDATA. + */ + if (S_ISDIR(file->mode) && + strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) + { + Oid tblspcOid; + char tmp_rel_path[MAXPGPATH]; + + /* + * Valid path for the tablespace is + * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + */ + if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + return false; + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s", + &tblspcOid, tmp_rel_path); + if (sscanf_res == 0) + return false; + } + + if (path_is_prefix_of_path("global", rel_path)) + { + file->tblspcOid = GLOBALTABLESPACE_OID; + + if (S_ISDIR(file->mode) && strcmp(file->name, "global") == 0) + file->is_database = true; + } + else if (path_is_prefix_of_path("base", rel_path)) + { + file->tblspcOid = DEFAULTTABLESPACE_OID; + + sscanf(rel_path, "base/%u/", &(file->dbOid)); + + if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) + file->is_database = true; + } + else if (path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + { + char tmp_rel_path[MAXPGPATH]; + + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", + &(file->tblspcOid), tmp_rel_path, + &(file->dbOid)); + + if (sscanf_res == 3 && S_ISDIR(file->mode) && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + file->is_database = true; + } + + /* Do not backup ptrack_init files */ + if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) + return false; + + /* + * Check files located inside database directories including directory + * 'global' + */ + if (S_ISREG(file->mode) && file->tblspcOid != 0 && + file->name && file->name[0]) + { + if (strcmp(file->name, "pg_internal.init") == 0) + return false; + /* Do not backup temp files */ + else if (file->name[0] == 't' && isdigit(file->name[1])) + return false; + else if (isdigit(file->name[0])) + { + char *fork_name; + int len; + char suffix[MAXPGPATH]; + + fork_name = strstr(file->name, "_"); + if (fork_name) + { + /* Auxiliary fork of the relfile */ + sscanf(file->name, "%u_%s", &(file->relOid), file->forkName); + + /* Do not backup ptrack files */ + if (strcmp(file->forkName, "ptrack") == 0) + return false; + } + else + { + len = strlen(file->name); + /* reloid.cfm */ + if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) + return true; + + sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), + &(file->segno), suffix); + if (sscanf_res == 0) + elog(ERROR, "Cannot parse file name \"%s\"", file->name); + else if (sscanf_res == 1 || sscanf_res == 2) + file->is_datafile = true; + } + } + } + + return true; +} + +/* + * List files in "root" directory. If "exclude" is true do not add into "files" + * files from pgdata_exclude_files and directories from pgdata_exclude_dir. + */ +static void +dir_list_file_internal(parray *files, const char *root, pgFile *parent, + bool exclude, bool omit_symlink, parray *black_list) +{ + DIR *dir; + struct dirent *dent; + + if (!S_ISDIR(parent->mode)) + elog(ERROR, "\"%s\" is not a directory", parent->path); + + /* Open directory and list contents */ + dir = opendir(parent->path); + if (dir == NULL) + { + if (errno == ENOENT) + { + /* Maybe the directory was removed */ + return; + } + elog(ERROR, "cannot open directory \"%s\": %s", + parent->path, strerror(errno)); + } + + errno = 0; + while ((dent = readdir(dir))) + { + pgFile *file; + char child[MAXPGPATH]; + + join_path_components(child, parent->path, dent->d_name); + + file = pgFileNew(child, omit_symlink); + if (file == NULL) + continue; + + /* Skip entries point current dir or parent dir */ + if (S_ISDIR(file->mode) && + (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) + { + pgFileFree(file); + continue; + } + + /* + * Add only files, directories and links. Skip sockets and other + * unexpected file formats. + */ + if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) + { + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + pgFileFree(file); + continue; + } + + /* Skip if the directory is in black_list defined by user */ + if (black_list && parray_bsearch(black_list, file->path, + BlackListCompare)) + { + elog(LOG, "Skip \"%s\": it is in the user's black list", file->path); + pgFileFree(file); + continue; + } + + /* We add the directory anyway */ + if (S_ISDIR(file->mode)) + parray_append(files, file); + + if (exclude && !dir_check_file(root, file)) + { + if (S_ISREG(file->mode)) + pgFileFree(file); + /* Skip */ + continue; + } + + /* At least add the file */ + if (S_ISREG(file->mode)) + parray_append(files, file); + + /* + * If the entry is a directory call dir_list_file_internal() + * recursively. + */ + if (S_ISDIR(file->mode)) + dir_list_file_internal(files, root, file, exclude, omit_symlink, + black_list); + } + + if (errno && errno != ENOENT) + { + int errno_tmp = errno; + closedir(dir); + elog(ERROR, "cannot read directory \"%s\": %s", + parent->path, strerror(errno_tmp)); + } + closedir(dir); +} + +/* + * List data directories excluding directories from + * pgdata_exclude_dir array. + * + * **is_root** is a little bit hack. We exclude only first level of directories + * and on the first level we check all files and directories. + */ +static void +list_data_directories(parray *files, const char *path, bool is_root, + bool exclude) +{ + DIR *dir; + struct dirent *dent; + int prev_errno; + bool has_child_dirs = false; + + /* open directory and list contents */ + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + bool skip = false; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + /* Check for exclude for the first level of listing */ + if (is_root && exclude) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++) + { + if (strcmp(dent->d_name, pgdata_exclude_dir[i]) == 0) + { + skip = true; + break; + } + } + } + if (skip) + continue; + + has_child_dirs = true; + list_data_directories(files, child, false, exclude); + } + + /* List only full and last directories */ + if (!is_root && !has_child_dirs) + { + pgFile *dir; + + dir = pgFileNew(path, false); + parray_append(files, dir); + } + + prev_errno = errno; + closedir(dir); + + if (prev_errno && prev_errno != ENOENT) + elog(ERROR, "cannot read directory \"%s\": %s", + path, strerror(prev_errno)); +} + +/* + * Save create directory path into memory. We can use it in next page restore to + * not raise the error "restore tablespace destination is not empty" in + * create_data_directories(). + */ +static void +set_tablespace_created(const char *link, const char *dir) +{ + TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); + + strcpy(cell->link_name, link); + strcpy(cell->linked_dir, dir); + cell->next = NULL; + + if (tablespace_created_dirs.tail) + tablespace_created_dirs.tail->next = cell; + else + tablespace_created_dirs.head = cell; + tablespace_created_dirs.tail = cell; +} + +/* + * Retrieve tablespace path, either relocated or original depending on whether + * -T was passed or not. + * + * Copy of function get_tablespace_mapping() from pg_basebackup.c. + */ +static const char * +get_tablespace_mapping(const char *dir) +{ + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(dir, cell->old_dir) == 0) + return cell->new_dir; + + return dir; +} + +/* + * Is directory was created when symlink was created in restore_directories(). + */ +static const char * +get_tablespace_created(const char *link) +{ + TablespaceCreatedListCell *cell; + + for (cell = tablespace_created_dirs.head; cell; cell = cell->next) + if (strcmp(link, cell->link_name) == 0) + return cell->linked_dir; + + return NULL; +} + +/* + * Split argument into old_dir and new_dir and append to tablespace mapping + * list. + * + * Copy of function tablespace_list_append() from pg_basebackup.c. + */ +void +opt_tablespace_map(pgut_option *opt, const char *arg) +{ + TablespaceListCell *cell = pgut_new(TablespaceListCell); + char *dst; + char *dst_ptr; + const char *arg_ptr; + + dst_ptr = dst = cell->old_dir; + for (arg_ptr = arg; *arg_ptr; arg_ptr++) + { + if (dst_ptr - dst >= MAXPGPATH) + elog(ERROR, "directory name too long"); + + if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') + ; /* skip backslash escaping = */ + else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) + { + if (*cell->new_dir) + elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); + else + dst = dst_ptr = cell->new_dir; + } + else + *dst_ptr++ = *arg_ptr; + } + + if (!*cell->old_dir || !*cell->new_dir) + elog(ERROR, "invalid tablespace mapping format \"%s\", " + "must be \"OLDDIR=NEWDIR\"", arg); + + /* + * This check isn't absolutely necessary. But all tablespaces are created + * with absolute directories, so specifying a non-absolute path here would + * just never match, possibly confusing users. It's also good to be + * consistent with the new_dir check. + */ + if (!is_absolute_path(cell->old_dir)) + elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", + cell->old_dir); + + if (!is_absolute_path(cell->new_dir)) + elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", + cell->new_dir); + + if (tablespace_dirs.tail) + tablespace_dirs.tail->next = cell; + else + tablespace_dirs.head = cell; + tablespace_dirs.tail = cell; +} + +/* + * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise + * an error if target directories exist. + * + * If **extract_tablespaces** is true then try to extract tablespace data + * directories into their initial path using tablespace_map file. + */ +void +create_data_directories(const char *data_dir, const char *backup_dir, + bool extract_tablespaces) +{ + parray *dirs, + *links = NULL; + size_t i; + char backup_database_dir[MAXPGPATH], + to_path[MAXPGPATH]; + + dirs = parray_new(); + if (extract_tablespaces) + { + links = parray_new(); + read_tablespace_map(links, backup_dir); + } + + join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); + list_data_directories(dirs, backup_database_dir, true, false); + + elog(LOG, "restore directories and symlinks..."); + + for (i = 0; i < parray_num(dirs); i++) + { + pgFile *dir = (pgFile *) parray_get(dirs, i); + char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); + + Assert(S_ISDIR(dir->mode)); + + /* Try to create symlink and linked directory if necessary */ + if (extract_tablespaces && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) + { + char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), + *link_sep, + *tmp_ptr; + char link_name[MAXPGPATH]; + pgFile **link; + + /* Extract link name from relative path */ + link_sep = first_dir_separator(link_ptr); + if (link_sep != NULL) + { + int len = link_sep - link_ptr; + strncpy(link_name, link_ptr, len); + link_name[len] = '\0'; + } + else + goto create_directory; + + tmp_ptr = dir->path; + dir->path = link_name; + /* Search only by symlink name without path */ + link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); + dir->path = tmp_ptr; + + if (link) + { + const char *linked_path = get_tablespace_mapping((*link)->linked); + const char *dir_created; + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + /* Check if linked directory was created earlier */ + dir_created = get_tablespace_created(link_name); + if (dir_created) + { + /* + * If symlink and linked directory were created do not + * create it second time. + */ + if (strcmp(dir_created, linked_path) == 0) + { + /* + * Create rest of directories. + * First check is there any directory name after + * separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + else + continue; + } + else + elog(ERROR, "tablespace directory \"%s\" of page backup does not " + "match with previous created tablespace directory \"%s\" of symlink \"%s\"", + linked_path, dir_created, link_name); + } + + /* + * This check was done in check_tablespace_mapping(). But do + * it again. + */ + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + + if (link_sep) + elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", + linked_path, + (int) (link_sep - relative_ptr), relative_ptr); + else + elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", + linked_path, relative_ptr); + + /* Firstly, create linked directory */ + dir_create_dir(linked_path, DIR_PERMISSION); + + join_path_components(to_path, data_dir, PG_TBLSPC_DIR); + /* Create pg_tblspc directory just in case */ + dir_create_dir(to_path, DIR_PERMISSION); + + /* Secondly, create link */ + join_path_components(to_path, to_path, link_name); + if (symlink(linked_path, to_path) < 0) + elog(ERROR, "could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + /* Save linked directory */ + set_tablespace_created(link_name, linked_path); + + /* + * Create rest of directories. + * First check is there any directory name after separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + + continue; + } + } + +create_directory: + elog(LOG, "create directory \"%s\"", relative_ptr); + + /* This is not symlink, create directory */ + join_path_components(to_path, data_dir, relative_ptr); + dir_create_dir(to_path, DIR_PERMISSION); + } + + if (extract_tablespaces) + { + parray_walk(links, pgFileFree); + parray_free(links); + } + + parray_walk(dirs, pgFileFree); + parray_free(dirs); +} + +/* + * Read names of symbolik names of tablespaces with links to directories from + * tablespace_map or tablespace_map.txt. + */ +void +read_tablespace_map(parray *files, const char *backup_dir) +{ + FILE *fp; + char db_path[MAXPGPATH], + map_path[MAXPGPATH]; + char buf[MAXPGPATH * 2]; + + join_path_components(db_path, backup_dir, DATABASE_DIR); + join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); + + /* Exit if database/tablespace_map doesn't exist */ + if (!fileExists(map_path)) + { + elog(LOG, "there is no file tablespace_map"); + return; + } + + fp = fopen(map_path, "rt"); + if (fp == NULL) + elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno)); + + while (fgets(buf, lengthof(buf), fp)) + { + char link_name[MAXPGPATH], + path[MAXPGPATH]; + pgFile *file; + + if (sscanf(buf, "%1023s %1023s", link_name, path) != 2) + elog(ERROR, "invalid format found in \"%s\"", map_path); + + file = pgut_new(pgFile); + memset(file, 0, sizeof(pgFile)); + + file->path = pgut_malloc(strlen(link_name) + 1); + strcpy(file->path, link_name); + + file->linked = pgut_malloc(strlen(path) + 1); + strcpy(file->linked, path); + + parray_append(files, file); + } + + parray_qsort(files, pgFileCompareLinked); + fclose(fp); +} + +/* + * Check that all tablespace mapping entries have correct linked directory + * paths. Linked directories must be empty or do not exist. + * + * If tablespace-mapping option is supplied, all OLDDIR entries must have + * entries in tablespace_map file. + */ +void +check_tablespace_mapping(pgBackup *backup) +{ + char this_backup_path[MAXPGPATH]; + parray *links; + size_t i; + TablespaceListCell *cell; + pgFile *tmp_file = pgut_new(pgFile); + + links = parray_new(); + + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, this_backup_path); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "check tablespace directories of backup %s", + base36enc(backup->start_time)); + + /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ + for (cell = tablespace_dirs.head; cell; cell = cell->next) + { + tmp_file->linked = cell->old_dir; + + if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) + elog(ERROR, "--tablespace-mapping option's old directory " + "doesn't have an entry in tablespace_map file: \"%s\"", + cell->old_dir); + } + + /* 2 - all linked directories must be empty */ + for (i = 0; i < parray_num(links); i++) + { + pgFile *link = (pgFile *) parray_get(links, i); + const char *linked_path = link->linked; + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(link->linked, cell->old_dir) == 0) + { + linked_path = cell->new_dir; + break; + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + } + + free(tmp_file); + parray_walk(links, pgFileFree); + parray_free(links); +} + +/* + * Print backup content list. + */ +void +print_file_list(FILE *out, const parray *files, const char *root) +{ + size_t i; + + /* print each file in the list */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *path = file->path; + + /* omit root directory portion */ + if (root && strstr(path, root) == path) + path = GetRelativePath(path, root); + + fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"is_cfs\":\"%u\", \"crc\":\"%u\", " + "\"compress_alg\":\"%s\"", + path, file->write_size, file->mode, + file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, + deparse_compress_alg(file->compress_alg)); + + if (file->is_datafile) + fprintf(out, ",\"segno\":\"%d\"", file->segno); + +#ifndef WIN32 + if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif + fprintf(out, ",\"linked\":\"%s\"", file->linked); + + if (file->n_blocks != BLOCKNUM_INVALID) + fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); + + fprintf(out, "}\n"); + } +} + +/* Parsing states for get_control_value() */ +#define CONTROL_WAIT_NAME 1 +#define CONTROL_INNAME 2 +#define CONTROL_WAIT_COLON 3 +#define CONTROL_WAIT_VALUE 4 +#define CONTROL_INVALUE 5 +#define CONTROL_WAIT_NEXT_NAME 6 + +/* + * Get value from json-like line "str" of backup_content.control file. + * + * The line has the following format: + * {"name1":"value1", "name2":"value2"} + * + * The value will be returned to "value_str" as string if it is not NULL. If it + * is NULL the value will be returned to "value_int64" as int64. + * + * Returns true if the value was found in the line. + */ +static bool +get_control_value(const char *str, const char *name, + char *value_str, int64 *value_int64, bool is_mandatory) +{ + int state = CONTROL_WAIT_NAME; + char *name_ptr = (char *) name; + char *buf = (char *) str; + char buf_int64[32], /* Buffer for "value_int64" */ + *buf_int64_ptr = buf_int64; + + /* Set default values */ + if (value_str) + *value_str = '\0'; + else if (value_int64) + *value_int64 = 0; + + while (*buf) + { + switch (state) + { + case CONTROL_WAIT_NAME: + if (*buf == '"') + state = CONTROL_INNAME; + else if (IsAlpha(*buf)) + goto bad_format; + break; + case CONTROL_INNAME: + /* Found target field. Parse value. */ + if (*buf == '"') + state = CONTROL_WAIT_COLON; + /* Check next field */ + else if (*buf != *name_ptr) + { + name_ptr = (char *) name; + state = CONTROL_WAIT_NEXT_NAME; + } + else + name_ptr++; + break; + case CONTROL_WAIT_COLON: + if (*buf == ':') + state = CONTROL_WAIT_VALUE; + else if (!IsSpace(*buf)) + goto bad_format; + break; + case CONTROL_WAIT_VALUE: + if (*buf == '"') + { + state = CONTROL_INVALUE; + buf_int64_ptr = buf_int64; + } + else if (IsAlpha(*buf)) + goto bad_format; + break; + case CONTROL_INVALUE: + /* Value was parsed, exit */ + if (*buf == '"') + { + if (value_str) + { + *value_str = '\0'; + } + else if (value_int64) + { + /* Length of buf_uint64 should not be greater than 31 */ + if (buf_int64_ptr - buf_int64 >= 32) + elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); + + *buf_int64_ptr = '\0'; + if (!parse_int64(buf_int64, value_int64, 0)) + goto bad_format; + } + + return true; + } + else + { + if (value_str) + { + *value_str = *buf; + value_str++; + } + else + { + *buf_int64_ptr = *buf; + buf_int64_ptr++; + } + } + break; + case CONTROL_WAIT_NEXT_NAME: + if (*buf == ',') + state = CONTROL_WAIT_NAME; + break; + default: + /* Should not happen */ + break; + } + + buf++; + } + + /* There is no close quotes */ + if (state == CONTROL_INNAME || state == CONTROL_INVALUE) + goto bad_format; + + /* Did not find target field */ + if (is_mandatory) + elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); + return false; + +bad_format: + elog(ERROR, "%s file has invalid format in line %s", + DATABASE_FILE_LIST, str); + return false; /* Make compiler happy */ +} + +/* + * Construct parray of pgFile from the backup content list. + * If root is not NULL, path will be absolute path. + */ +parray * +dir_read_file_list(const char *root, const char *file_txt) +{ + FILE *fp; + parray *files; + char buf[MAXPGPATH * 2]; + + fp = fopen(file_txt, "rt"); + if (fp == NULL) + elog(errno == ENOENT ? ERROR : ERROR, + "cannot open \"%s\": %s", file_txt, strerror(errno)); + + files = parray_new(); + + while (fgets(buf, lengthof(buf), fp)) + { + char path[MAXPGPATH]; + char filepath[MAXPGPATH]; + char linked[MAXPGPATH]; + char compress_alg_string[MAXPGPATH]; + int64 write_size, + mode, /* bit length of mode_t depends on platforms */ + is_datafile, + is_cfs, + crc, + segno, + n_blocks; + pgFile *file; + + get_control_value(buf, "path", path, NULL, true); + get_control_value(buf, "size", NULL, &write_size, true); + get_control_value(buf, "mode", NULL, &mode, true); + get_control_value(buf, "is_datafile", NULL, &is_datafile, true); + get_control_value(buf, "is_cfs", NULL, &is_cfs, false); + get_control_value(buf, "crc", NULL, &crc, true); + get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); + + if (root) + join_path_components(filepath, root, path); + else + strcpy(filepath, path); + + file = pgFileInit(filepath); + + file->write_size = (int64) write_size; + file->mode = (mode_t) mode; + file->is_datafile = is_datafile ? true : false; + file->is_cfs = is_cfs ? true : false; + file->crc = (pg_crc32) crc; + file->compress_alg = parse_compress_alg(compress_alg_string); + + /* + * Optional fields + */ + + if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) + file->linked = pgut_strdup(linked); + + if (get_control_value(buf, "segno", NULL, &segno, false)) + file->segno = (int) segno; + + if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) + file->n_blocks = (int) n_blocks; + + parray_append(files, file); + } + + fclose(fp); + return files; +} + +/* + * Check if directory empty. + */ +bool +dir_is_empty(const char *path) +{ + DIR *dir; + struct dirent *dir_ent; + + dir = opendir(path); + if (dir == NULL) + { + /* Directory in path doesn't exist */ + if (errno == ENOENT) + return true; + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + } + + errno = 0; + while ((dir_ent = readdir(dir))) + { + /* Skip entries point current dir or parent dir */ + if (strcmp(dir_ent->d_name, ".") == 0 || + strcmp(dir_ent->d_name, "..") == 0) + continue; + + /* Directory is not empty */ + closedir(dir); + return false; + } + if (errno) + elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno)); + + closedir(dir); + + return true; +} + +/* + * Return true if the path is a existing regular file. + */ +bool +fileExists(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1 && errno == ENOENT) + return false; + else if (!S_ISREG(buf.st_mode)) + return false; + else + return true; +} + +size_t +pgFileSize(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + + return buf.st_size; +} diff --git a/src/fetch.c b/src/fetch.c new file mode 100644 index 00000000..0d4dbdaa --- /dev/null +++ b/src/fetch.c @@ -0,0 +1,116 @@ +/*------------------------------------------------------------------------- + * + * fetch.c + * Functions for fetching files from PostgreSQL data directory + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "catalog/catalog.h" + +#include +#include +#include +#include +#include +#include + +#include "pg_probackup.h" + +/* + * Read a file into memory. The file to be read is /. + * The file contents are returned in a malloc'd buffer, and *filesize + * is set to the length of the file. + * + * The returned buffer is always zero-terminated; the size of the returned + * buffer is actually *filesize + 1. That's handy when reading a text file. + * This function can be used to read binary files as well, you can just + * ignore the zero-terminator in that case. + * + */ +char * +slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) +{ + int fd; + char *buffer; + struct stat statbuf; + char fullpath[MAXPGPATH]; + int len; + snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); + + if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + + if (fstat(fd, &statbuf) < 0) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + + len = statbuf.st_size; + + buffer = pg_malloc(len + 1); + + if (read(fd, buffer, len) != len) + { + if (safe) + return NULL; + else + elog(ERROR, "could not read file \"%s\": %s\n", + fullpath, strerror(errno)); + } + + close(fd); + + /* Zero-terminate the buffer. */ + buffer[len] = '\0'; + + if (filesize) + *filesize = len; + return buffer; +} + +/* + * Receive a single file as a malloc'd buffer. + */ +char * +fetchFile(PGconn *conn, const char *filename, size_t *filesize) +{ + PGresult *res; + char *result; + const char *params[1]; + int len; + + params[0] = filename; + res = pgut_execute_extended(conn, "SELECT pg_catalog.pg_read_binary_file($1)", + 1, params, false, false); + + /* sanity check the result set */ + if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) + elog(ERROR, "unexpected result set while fetching remote file \"%s\"", + filename); + + /* Read result to local variables */ + len = PQgetlength(res, 0, 0); + result = pg_malloc(len + 1); + memcpy(result, PQgetvalue(res, 0, 0), len); + result[len] = '\0'; + + PQclear(res); + *filesize = len; + + return result; +} diff --git a/src/help.c b/src/help.c new file mode 100644 index 00000000..dc9cc3d8 --- /dev/null +++ b/src/help.c @@ -0,0 +1,605 @@ +/*------------------------------------------------------------------------- + * + * help.c + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +static void help_init(void); +static void help_backup(void); +static void help_restore(void); +static void help_validate(void); +static void help_show(void); +static void help_delete(void); +static void help_merge(void); +static void help_set_config(void); +static void help_show_config(void); +static void help_add_instance(void); +static void help_del_instance(void); +static void help_archive_push(void); +static void help_archive_get(void); + +void +help_command(char *command) +{ + if (strcmp(command, "init") == 0) + help_init(); + else if (strcmp(command, "backup") == 0) + help_backup(); + else if (strcmp(command, "restore") == 0) + help_restore(); + else if (strcmp(command, "validate") == 0) + help_validate(); + else if (strcmp(command, "show") == 0) + help_show(); + else if (strcmp(command, "delete") == 0) + help_delete(); + else if (strcmp(command, "merge") == 0) + help_merge(); + else if (strcmp(command, "set-config") == 0) + help_set_config(); + else if (strcmp(command, "show-config") == 0) + help_show_config(); + else if (strcmp(command, "add-instance") == 0) + help_add_instance(); + else if (strcmp(command, "del-instance") == 0) + help_del_instance(); + else if (strcmp(command, "archive-push") == 0) + help_archive_push(); + else if (strcmp(command, "archive-get") == 0) + help_archive_get(); + else if (strcmp(command, "--help") == 0 + || strcmp(command, "help") == 0 + || strcmp(command, "-?") == 0 + || strcmp(command, "--version") == 0 + || strcmp(command, "version") == 0 + || strcmp(command, "-V") == 0) + printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); + else + printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); + exit(0); +} + +void +help_pg_probackup(void) +{ + printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + + printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); + + printf(_("\n %s version\n"), PROGRAM_NAME); + + printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); + + printf(_("\n %s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--archive-timeout=timeout]\n")); + + printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n")); + + printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); + printf(_(" [--restore-as-replica]\n")); + printf(_(" [--no-validate]\n")); + + printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); + printf(_(" [--timeline=timeline]\n")); + + printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n")); + + printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--wal] [-i backup-id | --expired]\n")); + printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id\n")); + + printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--overwrite]\n")); + + printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + + if ((PROGRAM_URL || PROGRAM_EMAIL)) + { + printf("\n"); + if (PROGRAM_URL) + printf("Read the website for details. <%s>\n", PROGRAM_URL); + if (PROGRAM_EMAIL) + printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); + } + exit(0); +} + +static void +help_init(void) +{ + printf(_("%s init -B backup-path\n\n"), PROGRAM_NAME); + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); +} + +static void +help_backup(void) +{ + printf(_("%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); + printf(_(" --stream stream the transaction log and include it in the backup\n")); + printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --backup-pg-log backup of pg_log directory\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --delete-expired delete backups expired according to current\n")); + printf(_(" retention policy after successful backup completion\n")); + printf(_(" --delete-wal remove redundant archived wal files\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress compress data files\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib', 'pglz', 'none' (default: zlib)\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); +} + +static void +help_restore(void) +{ + printf(_("%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); + printf(_(" [--restore-as-replica] [--no-validate]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" -i, --backup-id=backup-id backup to restore\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + + printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); + printf(_(" --recovery-target-action=pause|promote|shutdown\n")); + printf(_(" action the server should take once the recovery target is reached\n")); + printf(_(" (default: pause)\n")); + + printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); + printf(_(" to ease setting up a standby server\n")); + printf(_(" --no-validate disable backup validation during restore\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_validate(void) +{ + printf(_("%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to validate\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_show(void) +{ + printf(_("%s show -B backup-path\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name show info about specific intstance\n")); + printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); +} + +static void +help_delete(void) +{ + printf(_("%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-i backup-id | --expired] [--wal]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to delete\n")); + printf(_(" --expired delete backups expired according to current\n")); + printf(_(" retention policy\n")); + printf(_(" --wal remove unnecessary wal files in WAL ARCHIVE\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_merge(void) +{ + printf(_("%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id [-j num-threads] [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to merge\n")); + + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_set_config(void) +{ + printf(_("%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--archive-timeout=timeout]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); +} + +static void +help_show_config(void) +{ + printf(_("%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); +} + +static void +help_add_instance(void) +{ + printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" --instance=instance_name name of the new instance\n")); +} + +static void +help_del_instance(void) +{ + printf(_("%s del-instance -B backup-path --instance=instance_name\n\n"), PROGRAM_NAME); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); +} + +static void +help_archive_push(void) +{ + printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--overwrite]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative path name of the WAL file on the server\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" name of the WAL file to retrieve from the server\n")); + printf(_(" --compress compress WAL file during archiving\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + printf(_(" --overwrite overwrite archived WAL file\n")); +} + +static void +help_archive_get(void) +{ + printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative destination path name of the WAL file on the server\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" name of the WAL file to retrieve from the archive\n")); +} diff --git a/src/init.c b/src/init.c new file mode 100644 index 00000000..cd559cb4 --- /dev/null +++ b/src/init.c @@ -0,0 +1,108 @@ +/*------------------------------------------------------------------------- + * + * init.c: - initialize backup catalog. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +/* + * Initialize backup catalog. + */ +int +do_init(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + int results; + + results = pg_check_dir(backup_path); + if (results == 4) /* exists and not empty*/ + elog(ERROR, "backup catalog already exist and it's not empty"); + else if (results == -1) /*trouble accessing directory*/ + { + int errno_tmp = errno; + elog(ERROR, "cannot open backup catalog directory \"%s\": %s", + backup_path, strerror(errno_tmp)); + } + + /* create backup catalog root directory */ + dir_create_dir(backup_path, DIR_PERMISSION); + + /* create backup catalog data directory */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir_create_dir(path, DIR_PERMISSION); + + /* create backup catalog wal directory */ + join_path_components(arclog_path_dir, backup_path, "wal"); + dir_create_dir(arclog_path_dir, DIR_PERMISSION); + + elog(INFO, "Backup catalog '%s' successfully inited", backup_path); + return 0; +} + +int +do_add_instance(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + struct stat st; + pgBackupConfig *config = pgut_new(pgBackupConfig); + + /* PGDATA is always required */ + if (pgdata == NULL) + elog(ERROR, "Required parameter not specified: PGDATA " + "(-D, --pgdata)"); + + /* Read system_identifier from PGDATA */ + system_identifier = get_system_identifier(pgdata); + /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ + xlog_seg_size = get_xlog_seg_size(pgdata); + + /* Ensure that all root directories already exist */ + if (access(backup_path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", backup_path); + + join_path_components(path, backup_path, BACKUPS_DIR); + if (access(path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", path); + + join_path_components(arclog_path_dir, backup_path, "wal"); + if (access(arclog_path_dir, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", arclog_path_dir); + + /* Create directory for data files of this specific instance */ + if (stat(backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "instance '%s' already exists", backup_instance_path); + dir_create_dir(backup_instance_path, DIR_PERMISSION); + + /* + * Create directory for wal files of this specific instance. + * Existence check is extra paranoid because if we don't have such a + * directory in data dir, we shouldn't have it in wal as well. + */ + if (stat(arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "arclog_path '%s' already exists", arclog_path); + dir_create_dir(arclog_path, DIR_PERMISSION); + + /* + * Wite initial config. system-identifier and pgdata are set in + * init subcommand and will never be updated. + */ + pgBackupConfigInit(config); + config->system_identifier = system_identifier; + config->xlog_seg_size = xlog_seg_size; + config->pgdata = pgdata; + writeBackupCatalogConfigFile(config); + + elog(INFO, "Instance '%s' successfully inited", instance_name); + return 0; +} diff --git a/src/merge.c b/src/merge.c new file mode 100644 index 00000000..979a1729 --- /dev/null +++ b/src/merge.c @@ -0,0 +1,526 @@ +/*------------------------------------------------------------------------- + * + * merge.c: merge FULL and incremental backups + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +typedef struct +{ + parray *to_files; + parray *files; + + pgBackup *to_backup; + pgBackup *from_backup; + + const char *to_root; + const char *from_root; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} merge_files_arg; + +static void merge_backups(pgBackup *backup, pgBackup *next_backup); +static void *merge_files(void *arg); + +/* + * Implementation of MERGE command. + * + * - Find target and its parent full backup + * - Merge data files of target, parent and and intermediate backups + * - Remove unnecessary files, which doesn't exist in the target backup anymore + */ +void +do_merge(time_t backup_id) +{ + parray *backups; + pgBackup *dest_backup = NULL; + pgBackup *full_backup = NULL; + time_t prev_parent = INVALID_BACKUP_ID; + int i; + int dest_backup_idx = 0; + int full_backup_idx = 0; + + if (backup_id == INVALID_BACKUP_ID) + elog(ERROR, "required parameter is not specified: --backup-id"); + + if (instance_name == NULL) + elog(ERROR, "required parameter is not specified: --instance"); + + elog(LOG, "Merge started"); + + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find destination and parent backups */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->start_time > backup_id) + continue; + else if (backup->start_time == backup_id && !dest_backup) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + if (backup->backup_mode == BACKUP_MODE_FULL) + elog(ERROR, "Backup %s if full backup", + base36enc(backup->start_time)); + + dest_backup = backup; + dest_backup_idx = i; + } + else + { + Assert(dest_backup); + + if (backup->start_time != prev_parent) + continue; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Skipping backup %s, because it has non-valid status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + /* If we already found dest_backup, look for full backup */ + if (dest_backup && backup->backup_mode == BACKUP_MODE_FULL) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Parent full backup %s for the given backup %s has status: %s", + base36enc_dup(backup->start_time), + base36enc_dup(dest_backup->start_time), + status2str(backup->status)); + + full_backup = backup; + full_backup_idx = i; + + /* Found target and full backups, so break the loop */ + break; + } + } + + prev_parent = backup->parent_backup; + } + + if (dest_backup == NULL) + elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); + if (full_backup == NULL) + elog(ERROR, "Parent full backup for the given backup %s was not found", + base36enc(backup_id)); + + Assert(full_backup_idx != dest_backup_idx); + + /* + * Found target and full backups, merge them and intermediate backups + */ + for (i = full_backup_idx; i > dest_backup_idx; i--) + { + pgBackup *to_backup = (pgBackup *) parray_get(backups, i); + pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); + + merge_backups(to_backup, from_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(LOG, "Merge completed"); +} + +/* + * Merge two backups data files using threads. + * - move instance files from from_backup to to_backup + * - remove unnecessary directories and files from to_backup + * - update metadata of from_backup, it becames FULL backup + */ +static void +merge_backups(pgBackup *to_backup, pgBackup *from_backup) +{ + char *to_backup_id = base36enc_dup(to_backup->start_time), + *from_backup_id = base36enc_dup(from_backup->start_time); + char to_backup_path[MAXPGPATH], + to_database_path[MAXPGPATH], + from_backup_path[MAXPGPATH], + from_database_path[MAXPGPATH], + control_file[MAXPGPATH]; + parray *files, + *to_files; + pthread_t *threads; + merge_files_arg *threads_args; + int i; + bool merge_isok = true; + + elog(LOG, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + + to_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(to_backup); + + from_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(from_backup); + + /* + * Make backup paths. + */ + pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); + pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), + DATABASE_DIR); + pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); + pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), + DATABASE_DIR); + + create_data_directories(to_database_path, from_backup_path, false); + + /* + * Get list of files which will be modified or removed. + */ + pgBackupGetPath(to_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + to_files = dir_read_file_list(from_database_path, /* Use from_database_path + * so root path will be + * equal with 'files' */ + control_file); + /* To delete from leaf, sort in reversed order */ + parray_qsort(to_files, pgFileComparePathDesc); + /* + * Get list of files which need to be moved. + */ + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + files = dir_read_file_list(from_database_path, control_file); + /* sort by size for load balancing */ + parray_qsort(files, pgFileCompareSize); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + + /* Setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_init_flag(&file->lock); + } + + for (i = 0; i < num_threads; i++) + { + merge_files_arg *arg = &(threads_args[i]); + + arg->to_files = to_files; + arg->files = files; + arg->to_backup = to_backup; + arg->from_backup = from_backup; + arg->to_root = to_database_path; + arg->from_root = from_database_path; + /* By default there are some error */ + arg->ret = 1; + + elog(VERBOSE, "Start thread: %d", i); + + pthread_create(&threads[i], NULL, merge_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + merge_isok = false; + } + if (!merge_isok) + elog(ERROR, "Data files merging failed"); + + /* + * Files were copied into to_backup and deleted from from_backup. Remove + * remaining directories from from_backup. + */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (!S_ISDIR(file->mode)) + continue; + + if (rmdir(file->path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + file->path, strerror(errno)); + } + if (rmdir(from_database_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_database_path, strerror(errno)); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + BACKUP_CONTROL_FILE); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + if (rmdir(from_backup_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_backup_path, strerror(errno)); + + /* + * Delete files which are not in from_backup file list. + */ + for (i = 0; i < parray_num(to_files); i++) + { + pgFile *file = (pgFile *) parray_get(to_files, i); + + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + elog(LOG, "Deleted \"%s\"", file->path); + } + } + + /* + * Rename FULL backup directory. + */ + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Update to_backup metadata. + */ + pgBackupCopy(to_backup, from_backup); + /* Correct metadata */ + to_backup->backup_mode = BACKUP_MODE_FULL; + to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + /* Compute summary of size of regular files in the backup */ + to_backup->data_bytes = 0; + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (S_ISDIR(file->mode)) + to_backup->data_bytes += 4096; + /* Count the amount of the data actually copied */ + else if (S_ISREG(file->mode)) + to_backup->data_bytes += file->write_size; + } + /* compute size of wal files of this backup stored in the archive */ + if (!to_backup->stream) + to_backup->wal_bytes = xlog_seg_size * + (to_backup->stop_lsn / xlog_seg_size - + to_backup->start_lsn / xlog_seg_size + 1); + else + to_backup->wal_bytes = BYTES_INVALID; + + pgBackupWriteFileList(to_backup, files, from_database_path); + pgBackupWriteBackupControlFile(to_backup); + + /* Cleanup */ + pfree(threads_args); + pfree(threads); + + parray_walk(to_files, pgFileFree); + parray_free(to_files); + + parray_walk(files, pgFileFree); + parray_free(files); + + pfree(to_backup_id); + pfree(from_backup_id); +} + +/* + * Thread worker of merge_backups(). + */ +static void * +merge_files(void *arg) +{ + merge_files_arg *argument = (merge_files_arg *) arg; + pgBackup *to_backup = argument->to_backup; + pgBackup *from_backup = argument->from_backup; + char tmp_file_path[MAXPGPATH]; + int i, + num_files = parray_num(argument->files); + int to_root_len = strlen(argument->to_root); + + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + join_path_components(tmp_file_path, argument->to_root, "tmp"); + + for (i = 0; i < num_files; i++) + { + pgFile *file = (pgFile *) parray_get(argument->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during merging backups"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); + + /* + * Skip files which haven't changed since previous backup. But in case + * of DELTA backup we should consider n_blocks to truncate the target + * backup. + */ + if (file->write_size == BYTES_INVALID && + file->n_blocks == -1) + { + elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", + file->path); + + /* + * If the file wasn't changed in PAGE backup, retreive its + * write_size from previous FULL backup. + */ + if (S_ISREG(file->mode)) + { + pgFile **res_file; + + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + if (res_file && *res_file) + { + file->compress_alg = (*res_file)->compress_alg; + file->write_size = (*res_file)->write_size; + file->crc = (*res_file)->crc; + } + } + + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + + /* + * Move the file. We need to decompress it and compress again if + * necessary. + */ + elog(VERBOSE, "Moving file \"%s\", is_datafile %d, is_cfs %d", + file->path, file->is_database, file->is_cfs); + + if (file->is_datafile && !file->is_cfs) + { + char to_path_tmp[MAXPGPATH]; /* Path of target file */ + + join_path_components(to_path_tmp, argument->to_root, + file->path + to_root_len + 1); + + /* + * We need more complicate algorithm if target file exists and it is + * compressed. + */ + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + { + char *prev_path; + + /* Start the magic */ + + /* + * Merge files: + * - decompress first file + * - decompress second file and merge with first decompressed file + * - compress result file + */ + + elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", + tmp_file_path); + + prev_path = file->path; + /* + * We need to decompress target file only if it exists. + */ + if (fileExists(to_path_tmp)) + { + /* + * file->path points to the file in from_root directory. But we + * need the file in directory to_root. + */ + file->path = to_path_tmp; + + /* Decompress first/target file */ + restore_data_file(tmp_file_path, file, false, false); + + file->path = prev_path; + } + /* Merge second/source file with first/target file */ + restore_data_file(tmp_file_path, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + + elog(VERBOSE, "Compress file and save it to the directory \"%s\"", + argument->to_root); + + /* Again we need change path */ + file->path = tmp_file_path; + /* backup_data_file() requires file size to calculate nblocks */ + file->size = pgFileSize(file->path); + /* Now we can compress the file */ + backup_data_file(NULL, /* We shouldn't need 'arguments' here */ + to_path_tmp, file, + to_backup->start_lsn, + to_backup->backup_mode, + to_backup->compress_alg, + to_backup->compress_level); + + file->path = prev_path; + + /* We can remove temporary file now */ + if (unlink(tmp_file_path)) + elog(ERROR, "Could not remove temporary file \"%s\": %s", + tmp_file_path, strerror(errno)); + } + /* + * Otherwise merging algorithm is simpler. + */ + else + { + /* We can merge in-place here */ + restore_data_file(to_path_tmp, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + true); + + /* + * We need to calculate write_size, restore_data_file() doesn't + * do that. + */ + file->write_size = pgFileSize(to_path_tmp); + file->crc = pgFileGetCRC(to_path_tmp); + } + pgFileDelete(file); + } + else + move_file(argument->from_root, argument->to_root, file); + + if (file->write_size != BYTES_INVALID) + elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files merging is successful */ + argument->ret = 0; + + return NULL; +} diff --git a/src/parsexlog.c b/src/parsexlog.c new file mode 100644 index 00000000..297269b6 --- /dev/null +++ b/src/parsexlog.c @@ -0,0 +1,1039 @@ +/*------------------------------------------------------------------------- + * + * parsexlog.c + * Functions for reading Write-Ahead-Log + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#ifdef HAVE_LIBZ +#include +#endif + +#include "commands/dbcommands_xlog.h" +#include "catalog/storage_xlog.h" +#include "access/transam.h" +#include "utils/thread.h" + +/* + * RmgrNames is an array of resource manager names, to make error messages + * a bit nicer. + */ +#if PG_VERSION_NUM >= 100000 +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ + name, +#else +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \ + name, +#endif + +static const char *RmgrNames[RM_MAX_ID + 1] = { +#include "access/rmgrlist.h" +}; + +/* some from access/xact.h */ +/* + * XLOG allows to store some information in high 4 bits of log record xl_info + * field. We use 3 for the opcode, and one about an optional flag variable. + */ +#define XLOG_XACT_COMMIT 0x00 +#define XLOG_XACT_PREPARE 0x10 +#define XLOG_XACT_ABORT 0x20 +#define XLOG_XACT_COMMIT_PREPARED 0x30 +#define XLOG_XACT_ABORT_PREPARED 0x40 +#define XLOG_XACT_ASSIGNMENT 0x50 +/* free opcode 0x60 */ +/* free opcode 0x70 */ + +/* mask for filtering opcodes out of xl_info */ +#define XLOG_XACT_OPMASK 0x70 + +typedef struct xl_xact_commit +{ + TimestampTz xact_time; /* time of commit */ + + /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ + /* xl_xact_dbinfo follows if XINFO_HAS_DBINFO */ + /* xl_xact_subxacts follows if XINFO_HAS_SUBXACT */ + /* xl_xact_relfilenodes follows if XINFO_HAS_RELFILENODES */ + /* xl_xact_invals follows if XINFO_HAS_INVALS */ + /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ + /* xl_xact_origin follows if XINFO_HAS_ORIGIN, stored unaligned! */ +} xl_xact_commit; + +typedef struct xl_xact_abort +{ + TimestampTz xact_time; /* time of abort */ + + /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ + /* No db_info required */ + /* xl_xact_subxacts follows if HAS_SUBXACT */ + /* xl_xact_relfilenodes follows if HAS_RELFILENODES */ + /* No invalidation messages needed. */ + /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ +} xl_xact_abort; + +static void extractPageInfo(XLogReaderState *record); +static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); + +typedef struct XLogPageReadPrivate +{ + const char *archivedir; + TimeLineID tli; + uint32 xlog_seg_size; + + bool manual_switch; + bool need_switch; + + int xlogfile; + XLogSegNo xlogsegno; + char xlogpath[MAXPGPATH]; + bool xlogexists; + +#ifdef HAVE_LIBZ + gzFile gz_xlogfile; + char gz_xlogpath[MAXPGPATH]; +#endif +} XLogPageReadPrivate; + +/* An argument for a thread function */ +typedef struct +{ + int thread_num; + XLogPageReadPrivate private_data; + + XLogRecPtr startpoint; + XLogRecPtr endpoint; + XLogSegNo endSegNo; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} xlog_thread_arg; + +static int SimpleXLogPageRead(XLogReaderState *xlogreader, + XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, char *readBuf, + TimeLineID *pageTLI); +static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, + const char *archivedir, + TimeLineID tli, uint32 xlog_seg_size, + bool allocate_reader); +static void CleanupXLogPageRead(XLogReaderState *xlogreader); +static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, + int elevel); + +static XLogSegNo nextSegNoToRead = 0; +static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * extractPageMap() worker. + */ +static void * +doExtractPageMap(void *arg) +{ + xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; + XLogPageReadPrivate *private_data; + XLogReaderState *xlogreader; + XLogSegNo nextSegNo = 0; + char *errormsg; + + private_data = &extract_arg->private_data; +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(private_data->xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Start LSN of thread %d: %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + /* Switch WAL segment manually below without using SimpleXLogPageRead() */ + private_data->manual_switch = true; + + do + { + XLogRecord *record; + + if (interrupted) + elog(ERROR, "Interrupted during WAL reading"); + + record = XLogReadRecord(xlogreader, extract_arg->startpoint, &errormsg); + + if (record == NULL) + { + XLogRecPtr errptr; + + /* + * Try to switch to the next WAL segment. Usually + * SimpleXLogPageRead() does it by itself. But here we need to do it + * manually to support threads. + */ + if (private_data->need_switch) + { + private_data->need_switch = false; + + /* Critical section */ + pthread_lock(&wal_segment_mutex); + Assert(nextSegNoToRead); + private_data->xlogsegno = nextSegNoToRead; + nextSegNoToRead++; + pthread_mutex_unlock(&wal_segment_mutex); + + /* We reach the end */ + if (private_data->xlogsegno > extract_arg->endSegNo) + break; + + /* Adjust next record position */ + GetXLogRecPtr(private_data->xlogsegno, 0, + private_data->xlog_seg_size, + extract_arg->startpoint); + /* Skip over the page header */ + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Thread %d switched to LSN %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + continue; + } + + errptr = extract_arg->startpoint ? + extract_arg->startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(WARNING, "could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(WARNING, "could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + + /* + * If we don't have all WAL files from prev backup start_lsn to current + * start_lsn, we won't be able to build page map and PAGE backup will + * be incorrect. Stop it and throw an error. + */ + PrintXLogCorruptionMsg(private_data, ERROR); + } + + extractPageInfo(xlogreader); + + /* continue reading at next record */ + extract_arg->startpoint = InvalidXLogRecPtr; + + GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, + private_data->xlog_seg_size); + } while (nextSegNo <= extract_arg->endSegNo && + xlogreader->EndRecPtr < extract_arg->endpoint); + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + /* Extracting is successful */ + extract_arg->ret = 0; + return NULL; +} + +/* + * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the + * given timeline. Collect data blocks touched by the WAL records into a page map. + * + * If **prev_segno** is true then read all segments up to **endpoint** segment + * minus one. Else read all segments up to **endpoint** segment. + * + * Pagemap extracting is processed using threads. Eeach thread reads single WAL + * file. + */ +void +extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, bool prev_seg, + parray *files) +{ + int i; + int threads_need = 0; + XLogSegNo endSegNo; + bool extract_isok = true; + pthread_t *threads; + xlog_thread_arg *thread_args; + time_t start_time, + end_time; + + elog(LOG, "Compiling pagemap"); + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + + GetXLogSegNo(endpoint, endSegNo, seg_size); + if (prev_seg) + endSegNo--; + + nextSegNoToRead = 0; + time(&start_time); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + thread_args = (xlog_thread_arg *) palloc(sizeof(xlog_thread_arg)*num_threads); + + /* + * Initialize thread args. + * + * Each thread works with its own WAL segment and we need to adjust + * startpoint value for each thread. + */ + for (i = 0; i < num_threads; i++) + { + InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, + seg_size, false); + thread_args[i].thread_num = i; + + thread_args[i].startpoint = startpoint; + thread_args[i].endpoint = endpoint; + thread_args[i].endSegNo = endSegNo; + /* By default there is some error */ + thread_args[i].ret = 1; + + /* Adjust startpoint to the next thread */ + if (nextSegNoToRead == 0) + GetXLogSegNo(startpoint, nextSegNoToRead, seg_size); + + nextSegNoToRead++; + /* + * If we need to read less WAL segments than num_threads, create less + * threads. + */ + if (nextSegNoToRead > endSegNo) + break; + GetXLogRecPtr(nextSegNoToRead, 0, seg_size, startpoint); + /* Skip over the page header */ + startpoint += SizeOfXLogLongPHD; + + threads_need++; + } + + /* Run threads */ + for (i = 0; i < threads_need; i++) + { + elog(VERBOSE, "Start WAL reader thread: %d", i); + pthread_create(&threads[i], NULL, doExtractPageMap, &thread_args[i]); + } + + /* Wait for threads */ + for (i = 0; i < threads_need; i++) + { + pthread_join(threads[i], NULL); + if (thread_args[i].ret == 1) + extract_isok = false; + } + + pfree(threads); + pfree(thread_args); + + time(&end_time); + if (extract_isok) + elog(LOG, "Pagemap compiled, time elapsed %.0f sec", + difftime(end_time, start_time)); + else + elog(ERROR, "Pagemap compiling failed"); +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent state. + */ +static void +validate_backup_wal_from_start_to_stop(pgBackup *backup, + char *backup_xlog_path, TimeLineID tli, + uint32 xlog_seg_size) +{ + XLogRecPtr startpoint = backup->start_lsn; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + bool got_endpoint = false; + + xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, + xlog_seg_size, true); + + while (true) + { + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + /* Got WAL record at stop_lsn */ + if (xlogreader->ReadRecPtr == backup->stop_lsn) + { + got_endpoint = true; + break; + } + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (!got_endpoint) + { + PrintXLogCorruptionMsg(&private, WARNING); + + /* + * If we don't have WAL between start_lsn and stop_lsn, + * the backup is definitely corrupted. Update its status. + */ + backup->status = BACKUP_STATUS_CORRUPT; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "There are not enough WAL records to consistenly restore " + "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", + base36enc(backup->start_time), + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn), + (uint32) (backup->stop_lsn >> 32), + (uint32) (backup->stop_lsn)); + } + + /* clean */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent + * state. And check if we have in archive all files needed to restore the backup + * up to the given recovery target. + */ +void +validate_wal(pgBackup *backup, const char *archivedir, + time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 seg_size) +{ + XLogRecPtr startpoint = backup->start_lsn; + const char *backup_id; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + TransactionId last_xid = InvalidTransactionId; + TimestampTz last_time = 0; + char last_timestamp[100], + target_timestamp[100]; + bool all_wal = false; + char backup_xlog_path[MAXPGPATH]; + + /* We need free() this later */ + backup_id = base36enc(backup->start_time); + + if (!XRecOffIsValid(backup->start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", + (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), + backup_id); + + if (!XRecOffIsValid(backup->stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", + (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), + backup_id); + + /* + * Check that the backup has all wal files needed + * for recovery to consistent state. + */ + if (backup->stream) + { + snprintf(backup_xlog_path, sizeof(backup_xlog_path), "/%s/%s/%s/%s", + backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); + + validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, + seg_size); + } + else + validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli, + seg_size); + + if (backup->status == BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s WAL segments are corrupted", backup_id); + return; + } + /* + * If recovery target is provided check that we can restore backup to a + * recovery target time or xid. + */ + if (!TransactionIdIsValid(target_xid) && target_time == 0 && !XRecOffIsValid(target_lsn)) + { + /* Recovery target is not given so exit */ + elog(INFO, "Backup %s WAL segments are valid", backup_id); + return; + } + + /* + * If recovery target is provided, ensure that archive files exist in + * archive directory. + */ + if (dir_is_empty(archivedir)) + elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); + + /* + * Check if we have in archive all files needed to restore backup + * up to the given recovery target. + * In any case we cannot restore to the point before stop_lsn. + */ + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, + true); + + /* We can restore at least up to the backup end */ + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + last_xid = backup->recovery_xid; + + if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) + || (target_time != 0 && backup->recovery_time >= target_time) + || (XRecOffIsValid(target_lsn) && backup->stop_lsn >= target_lsn)) + all_wal = true; + + startpoint = backup->stop_lsn; + while (true) + { + bool timestamp_record; + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + timestamp_record = getRecordTimestamp(xlogreader, &last_time); + if (XLogRecGetXid(xlogreader) != InvalidTransactionId) + last_xid = XLogRecGetXid(xlogreader); + + /* Check target xid */ + if (TransactionIdIsValid(target_xid) && target_xid == last_xid) + { + all_wal = true; + break; + } + /* Check target time */ + else if (target_time != 0 && timestamp_record && timestamptz_to_time_t(last_time) >= target_time) + { + all_wal = true; + break; + } + /* If there are no target xid and target time */ + else if (!TransactionIdIsValid(target_xid) && target_time == 0 && + xlogreader->ReadRecPtr == backup->stop_lsn) + { + all_wal = true; + /* We don't stop here. We want to get last_xid and last_time */ + } + + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (last_time > 0) + time2iso(last_timestamp, lengthof(last_timestamp), + timestamptz_to_time_t(last_time)); + + /* There are all needed WAL records */ + if (all_wal) + elog(INFO, "backup validation completed successfully on time %s and xid " XID_FMT, + last_timestamp, last_xid); + /* Some needed WAL records are absent */ + else + { + PrintXLogCorruptionMsg(&private, WARNING); + + elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, + last_timestamp, last_xid); + + if (target_time > 0) + time2iso(target_timestamp, lengthof(target_timestamp), + target_time); + if (TransactionIdIsValid(target_xid) && target_time != 0) + elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, + target_timestamp, target_xid); + else if (TransactionIdIsValid(target_xid)) + elog(ERROR, "not enough WAL records to xid " XID_FMT, + target_xid); + else if (target_time != 0) + elog(ERROR, "not enough WAL records to time %s", + target_timestamp); + else if (XRecOffIsValid(target_lsn)) + elog(ERROR, "not enough WAL records to lsn %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + } + + /* clean */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); +} + +/* + * Read from archived WAL segments latest recovery time and xid. All necessary + * segments present at archive folder. We waited **stop_lsn** in + * pg_stop_backup(). + */ +bool +read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr start_lsn, XLogRecPtr stop_lsn, + time_t *recovery_time, TransactionId *recovery_xid) +{ + XLogRecPtr startpoint = stop_lsn; + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + bool res; + + if (!XRecOffIsValid(start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X", + (uint32) (start_lsn >> 32), (uint32) (start_lsn)); + + if (!XRecOffIsValid(stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + + /* Read records from stop_lsn down to start_lsn */ + do + { + XLogRecord *record; + TimestampTz last_time = 0; + char *errormsg; + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + XLogRecPtr errptr; + + errptr = startpoint ? startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(ERROR, "could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(ERROR, "could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + } + + /* Read previous record */ + startpoint = record->xl_prev; + + if (getRecordTimestamp(xlogreader, &last_time)) + { + *recovery_time = timestamptz_to_time_t(last_time); + *recovery_xid = XLogRecGetXid(xlogreader); + + /* Found timestamp in WAL record 'record' */ + res = true; + goto cleanup; + } + } while (startpoint >= start_lsn); + + /* Didn't find timestamp from WAL records between start_lsn and stop_lsn */ + res = false; + +cleanup: + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + +/* + * Check if there is a WAL segment file in 'archivedir' which contains + * 'target_lsn'. + */ +bool +wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, + TimeLineID target_tli, uint32 seg_size) +{ + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + char *errormsg; + bool res; + + if (!XRecOffIsValid(target_lsn)) + elog(ERROR, "Invalid target_lsn value %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + + xlogreader = InitXLogPageRead(&private, archivedir, target_tli, seg_size, + true); + + res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; + /* Didn't find 'target_lsn' and there is no error, return false */ + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf) +{ + int errnum; + const char *errmsg; + + errmsg = gzerror(gzf, &errnum); + if (errnum == Z_ERRNO) + return strerror(errno); + else + return errmsg; +} +#endif + +/* XLogreader callback function, to read a WAL page */ +static int +SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, char *readBuf, + TimeLineID *pageTLI) +{ + XLogPageReadPrivate *private_data; + uint32 targetPageOff; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + targetPageOff = targetPagePtr % private_data->xlog_seg_size; + + /* + * See if we need to switch to a new segment because the requested record + * is not in the currently open one. + */ + if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size)) + { + CleanupXLogPageRead(xlogreader); + /* + * Do not switch to next WAL segment in this function. Currently it is + * manually switched only in doExtractPageMap(). + */ + if (private_data->manual_switch) + { + private_data->need_switch = true; + return -1; + } + } + + GetXLogSegNo(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size); + + /* Try to switch to the next WAL segment */ + if (!private_data->xlogexists) + { + char xlogfname[MAXFNAMELEN]; + + GetXLogFileName(xlogfname, private_data->tli, private_data->xlogsegno, + private_data->xlog_seg_size); + snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", + private_data->archivedir, xlogfname); + + if (fileExists(private_data->xlogpath)) + { + elog(LOG, "Opening WAL segment \"%s\"", private_data->xlogpath); + + private_data->xlogexists = true; + private_data->xlogfile = open(private_data->xlogpath, + O_RDONLY | PG_BINARY, 0); + + if (private_data->xlogfile < 0) + { + elog(WARNING, "Could not open WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + } +#ifdef HAVE_LIBZ + /* Try to open compressed WAL segment */ + else + { + snprintf(private_data->gz_xlogpath, + sizeof(private_data->gz_xlogpath), "%s.gz", + private_data->xlogpath); + if (fileExists(private_data->gz_xlogpath)) + { + elog(LOG, "Opening compressed WAL segment \"%s\"", + private_data->gz_xlogpath); + + private_data->xlogexists = true; + private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, + "rb"); + if (private_data->gz_xlogfile == NULL) + { + elog(WARNING, "Could not open compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, strerror(errno)); + return -1; + } + } + } +#endif + + /* Exit without error if WAL segment doesn't exist */ + if (!private_data->xlogexists) + return -1; + } + + /* + * At this point, we have the right segment open. + */ + Assert(private_data->xlogexists); + + /* Read the requested page */ + if (private_data->xlogfile != -1) + { + if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) + { + elog(WARNING, "Could not seek in WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + + if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Could not read from WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + } +#ifdef HAVE_LIBZ + else + { + if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + { + elog(WARNING, "Could not seek in compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + + if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Could not read from compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + } +#endif + + *pageTLI = private_data->tli; + return XLOG_BLCKSZ; +} + +/* + * Initialize WAL segments reading. + */ +static XLogReaderState * +InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, + TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader) +{ + XLogReaderState *xlogreader = NULL; + + MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); + private_data->archivedir = archivedir; + private_data->tli = tli; + private_data->xlog_seg_size = xlog_seg_size; + private_data->xlogfile = -1; + + if (allocate_reader) + { +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + } + + return xlogreader; +} + +/* + * Cleanup after WAL segment reading. + */ +static void +CleanupXLogPageRead(XLogReaderState *xlogreader) +{ + XLogPageReadPrivate *private_data; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + if (private_data->xlogfile >= 0) + { + close(private_data->xlogfile); + private_data->xlogfile = -1; + } +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + { + gzclose(private_data->gz_xlogfile); + private_data->gz_xlogfile = NULL; + } +#endif + private_data->xlogexists = false; +} + +static void +PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) +{ + if (private_data->xlogpath[0] != 0) + { + /* + * XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status. + */ + if (!private_data->xlogexists) + elog(elevel, "WAL segment \"%s\" is absent", private_data->xlogpath); + else if (private_data->xlogfile != -1) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->xlogpath); +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->gz_xlogpath); +#endif + } +} + +/* + * Extract information about blocks modified in this record. + */ +static void +extractPageInfo(XLogReaderState *record) +{ + uint8 block_id; + RmgrId rmid = XLogRecGetRmid(record); + uint8 info = XLogRecGetInfo(record); + uint8 rminfo = info & ~XLR_INFO_MASK; + + /* Is this a special record type that I recognize? */ + + if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE) + { + /* + * New databases can be safely ignored. They would be completely + * copied if found. + */ + } + else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP) + { + /* + * An existing database was dropped. It is fine to ignore that + * they will be removed appropriately. + */ + } + else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE) + { + /* + * We can safely ignore these. The file will be removed when + * combining the backups in the case of differential on. + */ + } + else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE) + { + /* + * We can safely ignore these. When we compare the sizes later on, + * we'll notice that they differ, and copy the missing tail from + * source system. + */ + } + else if (info & XLR_SPECIAL_REL_UPDATE) + { + /* + * This record type modifies a relation file in some special way, but + * we don't recognize the type. That's bad - we don't know how to + * track that change. + */ + elog(ERROR, "WAL record modifies a relation, but record type is not recognized\n" + "lsn: %X/%X, rmgr: %s, info: %02X", + (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr), + RmgrNames[rmid], info); + } + + for (block_id = 0; block_id <= record->max_block_id; block_id++) + { + RelFileNode rnode; + ForkNumber forknum; + BlockNumber blkno; + + if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) + continue; + + /* We only care about the main fork; others are copied in toto */ + if (forknum != MAIN_FORKNUM) + continue; + + process_block_change(forknum, rnode, blkno); + } +} + +/* + * Extract timestamp from WAL record. + * + * If the record contains a timestamp, returns true, and saves the timestamp + * in *recordXtime. If the record type has no timestamp, returns false. + * Currently, only transaction commit/abort records and restore points contain + * timestamps. + */ +static bool +getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + uint8 xact_info = info & XLOG_XACT_OPMASK; + uint8 rmid = XLogRecGetRmid(record); + + if (rmid == RM_XLOG_ID && info == XLOG_RESTORE_POINT) + { + *recordXtime = ((xl_restore_point *) XLogRecGetData(record))->rp_time; + return true; + } + else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_COMMIT || + xact_info == XLOG_XACT_COMMIT_PREPARED)) + { + *recordXtime = ((xl_xact_commit *) XLogRecGetData(record))->xact_time; + return true; + } + else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_ABORT || + xact_info == XLOG_XACT_ABORT_PREPARED)) + { + *recordXtime = ((xl_xact_abort *) XLogRecGetData(record))->xact_time; + return true; + } + + return false; +} + diff --git a/src/pg_probackup.c b/src/pg_probackup.c new file mode 100644 index 00000000..a39ea5a8 --- /dev/null +++ b/src/pg_probackup.c @@ -0,0 +1,634 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.c: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "streamutil.h" +#include "utils/thread.h" + +#include +#include +#include +#include +#include +#include "pg_getopt.h" + +const char *PROGRAM_VERSION = "2.0.18"; +const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; +const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; + +/* directory options */ +char *backup_path = NULL; +char *pgdata = NULL; +/* + * path or to the data files in the backup catalog + * $BACKUP_PATH/backups/instance_name + */ +char backup_instance_path[MAXPGPATH]; +/* + * path or to the wal files in the backup catalog + * $BACKUP_PATH/wal/instance_name + */ +char arclog_path[MAXPGPATH] = ""; + +/* common options */ +static char *backup_id_string = NULL; +int num_threads = 1; +bool stream_wal = false; +bool progress = false; +#if PG_VERSION_NUM >= 100000 +char *replication_slot = NULL; +#endif + +/* backup options */ +bool backup_logs = false; +bool smooth_checkpoint; +bool is_remote_backup = false; +/* Wait timeout for WAL segment archiving */ +uint32 archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; +const char *master_db = NULL; +const char *master_host = NULL; +const char *master_port= NULL; +const char *master_user = NULL; +uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; + +/* restore options */ +static char *target_time; +static char *target_xid; +static char *target_lsn; +static char *target_inclusive; +static TimeLineID target_tli; +static bool target_immediate; +static char *target_name = NULL; +static char *target_action = NULL; + +static pgRecoveryTarget *recovery_target_options = NULL; + +bool restore_as_replica = false; +bool restore_no_validate = false; + +/* delete options */ +bool delete_wal = false; +bool delete_expired = false; +bool apply_to_all = false; +bool force_delete = false; + +/* retention options */ +uint32 retention_redundancy = 0; +uint32 retention_window = 0; + +/* compression options */ +CompressAlg compress_alg = COMPRESS_ALG_DEFAULT; +int compress_level = COMPRESS_LEVEL_DEFAULT; +bool compress_shortcut = false; + + +/* other options */ +char *instance_name; +uint64 system_identifier = 0; + +/* + * Starting from PostgreSQL 11 WAL segment size may vary. Prior to + * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. + */ +#if PG_VERSION_NUM >= 110000 +uint32 xlog_seg_size = 0; +#else +uint32 xlog_seg_size = XLOG_SEG_SIZE; +#endif + +/* archive push options */ +static char *wal_file_path; +static char *wal_file_name; +static bool file_overwrite = false; + +/* show options */ +ShowFormat show_format = SHOW_PLAIN; + +/* current settings */ +pgBackup current; +ProbackupSubcmd backup_subcmd = NO_CMD; + +static bool help_opt = false; + +static void opt_backup_mode(pgut_option *opt, const char *arg); +static void opt_log_level_console(pgut_option *opt, const char *arg); +static void opt_log_level_file(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); +static void opt_show_format(pgut_option *opt, const char *arg); + +static void compress_init(void); + +static pgut_option options[] = +{ + /* directory options */ + { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, + { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, + { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, + /* common options */ + { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, + { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, + { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, + { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMDLINE }, + /* backup options */ + { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, + { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, + { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, + { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, + { 'u', 11, "archive-timeout", &archive_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + { 'b', 12, "delete-wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 13, "delete-expired", &delete_expired, SOURCE_CMDLINE }, + { 's', 14, "master-db", &master_db, SOURCE_CMDLINE, }, + { 's', 15, "master-host", &master_host, SOURCE_CMDLINE, }, + { 's', 16, "master-port", &master_port, SOURCE_CMDLINE, }, + { 's', 17, "master-user", &master_user, SOURCE_CMDLINE, }, + { 'u', 18, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* TODO not completed feature. Make it unavailiable from user level + { 'b', 18, "remote", &is_remote_backup, SOURCE_CMDLINE, }, */ + /* restore options */ + { 's', 20, "time", &target_time, SOURCE_CMDLINE }, + { 's', 21, "xid", &target_xid, SOURCE_CMDLINE }, + { 's', 22, "inclusive", &target_inclusive, SOURCE_CMDLINE }, + { 'u', 23, "timeline", &target_tli, SOURCE_CMDLINE }, + { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, + { 'b', 24, "immediate", &target_immediate, SOURCE_CMDLINE }, + { 's', 25, "recovery-target-name", &target_name, SOURCE_CMDLINE }, + { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, + { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, + { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, + { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, + /* delete options */ + { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, + { 'b', 132, "all", &apply_to_all, SOURCE_CMDLINE }, + /* TODO not implemented yet */ + { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, + /* retention options */ + { 'u', 134, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, + { 'u', 135, "retention-window", &retention_window, SOURCE_CMDLINE }, + /* compression options */ + { 'f', 136, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 137, "compress-level", &compress_level, SOURCE_CMDLINE }, + { 'b', 138, "compress", &compress_shortcut, SOURCE_CMDLINE }, + /* logging options */ + { 'f', 140, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, + { 'f', 141, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, + { 's', 142, "log-filename", &log_filename, SOURCE_CMDLINE }, + { 's', 143, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, + { 's', 144, "log-directory", &log_directory, SOURCE_CMDLINE }, + { 'u', 145, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, + { 'u', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MIN }, + /* connection options */ + { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, + { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, + { 's', 'p', "pgport", &port, SOURCE_CMDLINE }, + { 's', 'U', "pguser", &username, SOURCE_CMDLINE }, + { 'B', 'w', "no-password", &prompt_password, SOURCE_CMDLINE }, + { 'b', 'W', "password", &force_password, SOURCE_CMDLINE }, + /* other options */ + { 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, + { 's', 151, "instance", &instance_name, SOURCE_CMDLINE }, +#if PG_VERSION_NUM >= 110000 + { 'u', 152, "xlog-seg-size", &xlog_seg_size, SOURCE_FILE_STRICT}, +#endif + /* archive-push options */ + { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, + { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, + { 'b', 162, "overwrite", &file_overwrite, SOURCE_CMDLINE }, + /* show options */ + { 'f', 170, "format", opt_show_format, SOURCE_CMDLINE }, + { 0 } +}; + +/* + * Entry point of pg_probackup command. + */ +int +main(int argc, char *argv[]) +{ + char *command = NULL, + *command_name; + /* Check if backup_path is directory. */ + struct stat stat_buf; + int rc; + + /* initialize configuration */ + pgBackupInit(¤t); + + PROGRAM_NAME = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], "pgscripts"); + +#if PG_VERSION_NUM >= 110000 + /* + * Reset WAL segment size, we will retreive it using RetrieveWalSegSize() + * later. + */ + WalSegSz = 0; +#endif + + /* + * Save main thread's tid. It is used call exit() in case of errors. + */ + main_tid = pthread_self(); + + /* Parse subcommands and non-subcommand options */ + if (argc > 1) + { + if (strcmp(argv[1], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[1], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[1], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[1], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[1], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[1], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[1], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[1], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[1], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "merge") == 0) + backup_subcmd = MERGE_CMD; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[1], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[1], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "--help") == 0 || + strcmp(argv[1], "-?") == 0 || + strcmp(argv[1], "help") == 0) + { + if (argc > 2) + help_command(argv[2]); + else + help_pg_probackup(); + } + else if (strcmp(argv[1], "--version") == 0 + || strcmp(argv[1], "version") == 0 + || strcmp(argv[1], "-V") == 0) + { +#ifdef PGPRO_VERSION + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif + exit(0); + } + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); + } + + if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); + + /* + * Make command string before getopt_long() will call. It permutes the + * content of argv. + */ + command_name = pstrdup(argv[1]); + if (backup_subcmd == BACKUP_CMD || + backup_subcmd == RESTORE_CMD || + backup_subcmd == VALIDATE_CMD || + backup_subcmd == DELETE_CMD || + backup_subcmd == MERGE_CMD) + { + int i, + len = 0, + allocated = 0; + + allocated = sizeof(char) * MAXPGPATH; + command = (char *) palloc(allocated); + + for (i = 0; i < argc; i++) + { + int arglen = strlen(argv[i]); + + if (arglen + len > allocated) + { + allocated *= 2; + command = repalloc(command, allocated); + } + + strncpy(command + len, argv[i], arglen); + len += arglen; + command[len++] = ' '; + } + + command[len] = '\0'; + } + + optind += 1; + /* Parse command line arguments */ + pgut_getopt(argc, argv, options); + + if (help_opt) + help_command(command_name); + + /* backup_path is required for all pg_probackup commands except help */ + if (backup_path == NULL) + { + /* + * If command line argument is not set, try to read BACKUP_PATH + * from environment variable + */ + backup_path = getenv("BACKUP_PATH"); + if (backup_path == NULL) + elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + } + canonicalize_path(backup_path); + + /* Ensure that backup_path is an absolute path */ + if (!is_absolute_path(backup_path)) + elog(ERROR, "-B, --backup-path must be an absolute path"); + + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + + /* command was initialized for a few commands */ + if (command) + { + elog_file(INFO, "command: %s", command); + + pfree(command); + command = NULL; + } + + /* Option --instance is required for all commands except init and show */ + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != VALIDATE_CMD) + { + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + } + + /* + * If --instance option was passed, construct paths for backup data and + * xlog files of this backup instance. + */ + if (instance_name) + { + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + + /* + * Ensure that requested backup instance exists. + * for all commands except init, which doesn't take this parameter + * and add-instance which creates new instance. + */ + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) + { + if (access(backup_instance_path, F_OK) != 0) + elog(ERROR, "Instance '%s' does not exist in this backup catalog", + instance_name); + } + } + + /* + * Read options from env variables or from config file, + * unless we're going to set them via set-config. + */ + if (instance_name && backup_subcmd != SET_CONFIG_CMD) + { + char path[MAXPGPATH]; + + /* Read environment variables */ + pgut_getopt_env(options); + + /* Read options from configuration file */ + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, true); + } + + /* Initialize logger */ + init_logger(backup_path); + + /* + * We have read pgdata path from command line or from configuration file. + * Ensure that pgdata is an absolute path. + */ + if (pgdata != NULL && !is_absolute_path(pgdata)) + elog(ERROR, "-D, --pgdata must be an absolute path"); + +#if PG_VERSION_NUM >= 110000 + /* Check xlog-seg-size option */ + if (instance_name && + backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != ADD_INSTANCE_CMD && !IsValidWalSegSize(xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", xlog_seg_size); +#endif + + /* Sanity check of --backup-id option */ + if (backup_id_string != NULL) + { + if (backup_subcmd != RESTORE_CMD && + backup_subcmd != VALIDATE_CMD && + backup_subcmd != DELETE_CMD && + backup_subcmd != MERGE_CMD && + backup_subcmd != SHOW_CMD) + elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", + command_name); + + current.backup_id = base36dec(backup_id_string); + if (current.backup_id == 0) + elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); + } + + /* Setup stream options. They are used in streamutil.c. */ + if (host != NULL) + dbhost = pstrdup(host); + if (port != NULL) + dbport = pstrdup(port); + if (username != NULL) + dbuser = pstrdup(username); + + /* setup exclusion list for file search */ + if (!backup_logs) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ + + /* Set 'pg_log' in first empty slot */ + pgdata_exclude_dir[i] = "pg_log"; + } + + if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) + { + /* parse all recovery target options into recovery_target_options structure */ + recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, + target_inclusive, target_tli, target_lsn, target_immediate, + target_name, target_action, restore_no_validate); + } + + if (num_threads < 1) + num_threads = 1; + + compress_init(); + + /* do actual operation */ + switch (backup_subcmd) + { + case ARCHIVE_PUSH_CMD: + return do_archive_push(wal_file_path, wal_file_name, file_overwrite); + case ARCHIVE_GET_CMD: + return do_archive_get(wal_file_path, wal_file_name); + case ADD_INSTANCE_CMD: + return do_add_instance(); + case DELETE_INSTANCE_CMD: + return do_delete_instance(); + case INIT_CMD: + return do_init(); + case BACKUP_CMD: + { + const char *backup_mode; + time_t start_time; + + start_time = time(NULL); + backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; + + elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", + PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, + stream_wal ? "true" : "false", is_remote_backup ? "true" : "false"); + + return do_backup(start_time); + } + case RESTORE_CMD: + return do_restore_or_validate(current.backup_id, + recovery_target_options, + true); + case VALIDATE_CMD: + if (current.backup_id == 0 && target_time == 0 && target_xid == 0) + return do_validate_all(); + else + return do_restore_or_validate(current.backup_id, + recovery_target_options, + false); + case SHOW_CMD: + return do_show(current.backup_id); + case DELETE_CMD: + if (delete_expired && backup_id_string) + elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); + if (!delete_expired && !delete_wal && !backup_id_string) + elog(ERROR, "You must specify at least one of the delete options: --expired |--wal |--backup_id"); + if (delete_wal && !delete_expired && !backup_id_string) + return do_retention_purge(); + if (delete_expired) + return do_retention_purge(); + else + return do_delete(current.backup_id); + case MERGE_CMD: + do_merge(current.backup_id); + break; + case SHOW_CONFIG_CMD: + return do_configure(true); + case SET_CONFIG_CMD: + return do_configure(false); + case NO_CMD: + /* Should not happen */ + elog(ERROR, "Unknown subcommand"); + } + + return 0; +} + +static void +opt_backup_mode(pgut_option *opt, const char *arg) +{ + current.backup_mode = parse_backup_mode(arg); +} + +static void +opt_log_level_console(pgut_option *opt, const char *arg) +{ + log_level_console = parse_log_level(arg); +} + +static void +opt_log_level_file(pgut_option *opt, const char *arg) +{ + log_level_file = parse_log_level(arg); +} + +static void +opt_show_format(pgut_option *opt, const char *arg) +{ + const char *v = arg; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0) + { + if (pg_strncasecmp("plain", v, len) == 0) + show_format = SHOW_PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + show_format = SHOW_JSON; + else + elog(ERROR, "Invalid show format \"%s\"", arg); + } + else + elog(ERROR, "Invalid show format \"%s\"", arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + compress_alg = parse_compress_alg(arg); +} + +/* + * Initialize compress and sanity checks for compress. + */ +static void +compress_init(void) +{ + /* Default algorithm is zlib */ + if (compress_shortcut) + compress_alg = ZLIB_COMPRESS; + + if (backup_subcmd != SET_CONFIG_CMD) + { + if (compress_level != COMPRESS_LEVEL_DEFAULT + && compress_alg == NOT_DEFINED_COMPRESS) + elog(ERROR, "Cannot specify compress-level option without compress-alg option"); + } + + if (compress_level < 0 || compress_level > 9) + elog(ERROR, "--compress-level value must be in the range from 0 to 9"); + + if (compress_level == 0) + compress_alg = NOT_DEFINED_COMPRESS; + + if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) + { +#ifndef HAVE_LIBZ + if (compress_alg == ZLIB_COMPRESS) + elog(ERROR, "This build does not support zlib compression"); + else +#endif + if (compress_alg == PGLZ_COMPRESS && num_threads > 1) + elog(ERROR, "Multithread backup does not support pglz compression"); + } +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h new file mode 100644 index 00000000..8f3a0fea --- /dev/null +++ b/src/pg_probackup.h @@ -0,0 +1,620 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.h: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef PG_PROBACKUP_H +#define PG_PROBACKUP_H + +#include "postgres_fe.h" + +#include +#include + +#include "access/timeline.h" +#include "access/xlogdefs.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum.h" +#include "utils/pg_crc.h" +#include "common/relpath.h" +#include "port.h" + +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif + +#include "utils/parray.h" +#include "utils/pgut.h" + +#include "datapagemap.h" + +# define PG_STOP_BACKUP_TIMEOUT 300 +/* + * Macro needed to parse ptrack. + * NOTE Keep those values syncronised with definitions in ptrack.h + */ +#define PTRACK_BITS_PER_HEAPBLOCK 1 +#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) + +/* Directory/File names */ +#define DATABASE_DIR "database" +#define BACKUPS_DIR "backups" +#if PG_VERSION_NUM >= 100000 +#define PG_XLOG_DIR "pg_wal" +#else +#define PG_XLOG_DIR "pg_xlog" +#endif +#define PG_TBLSPC_DIR "pg_tblspc" +#define PG_GLOBAL_DIR "global" +#define BACKUP_CONTROL_FILE "backup.control" +#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf" +#define BACKUP_CATALOG_PID "pg_probackup.pid" +#define DATABASE_FILE_LIST "backup_content.control" +#define PG_BACKUP_LABEL_FILE "backup_label" +#define PG_BLACK_LIST "black_list" +#define PG_TABLESPACE_MAP_FILE "tablespace_map" + +#define LOG_FILENAME_DEFAULT "pg_probackup.log" +#define LOG_DIRECTORY_DEFAULT "log" +/* Direcotry/File permission */ +#define DIR_PERMISSION (0700) +#define FILE_PERMISSION (0600) + +/* 64-bit xid support for PGPRO_EE */ +#ifndef PGPRO_EE +#define XID_FMT "%u" +#endif + +typedef enum CompressAlg +{ + NOT_DEFINED_COMPRESS = 0, + NONE_COMPRESS, + PGLZ_COMPRESS, + ZLIB_COMPRESS, +} CompressAlg; + +/* Information about single file (or dir) in backup */ +typedef struct pgFile +{ + char *name; /* file or directory name */ + mode_t mode; /* protection (file type and permission) */ + size_t size; /* size of the file */ + size_t read_size; /* size of the portion read (if only some pages are + backed up, it's different from size) */ + int64 write_size; /* size of the backed-up file. BYTES_INVALID means + that the file existed but was not backed up + because not modified since last backup. */ + /* we need int64 here to store '-1' value */ + pg_crc32 crc; /* CRC value of the file, regular file only */ + char *linked; /* path of the linked file */ + bool is_datafile; /* true if the file is PostgreSQL data file */ + char *path; /* absolute path of the file */ + Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ + Oid dbOid; /* dbOid extracted from path, if applicable */ + Oid relOid; /* relOid extracted from path, if applicable */ + char *forkName; /* forkName extracted from path, if applicable */ + int segno; /* Segment number for ptrack */ + int n_blocks; /* size of the file in blocks, readed during DELTA backup */ + bool is_cfs; /* Flag to distinguish files compressed by CFS*/ + bool is_database; + bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ + CompressAlg compress_alg; /* compression algorithm applied to the file */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ + datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, + * i.e. datafiles without _ptrack */ +} pgFile; + +/* Special values of datapagemap_t bitmapsize */ +#define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ + +/* Current state of backup */ +typedef enum BackupStatus +{ + BACKUP_STATUS_INVALID, /* the pgBackup is invalid */ + BACKUP_STATUS_OK, /* completed backup */ + BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ + BACKUP_STATUS_RUNNING, /* running backup */ + BACKUP_STATUS_MERGING, /* merging backups */ + BACKUP_STATUS_DELETING, /* data files are being deleted */ + BACKUP_STATUS_DELETED, /* data files have been deleted */ + BACKUP_STATUS_DONE, /* completed but not validated yet */ + BACKUP_STATUS_ORPHAN, /* backup validity is unknown but at least one parent backup is corrupted */ + BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ +} BackupStatus; + +typedef enum BackupMode +{ + BACKUP_MODE_INVALID = 0, + BACKUP_MODE_DIFF_PAGE, /* incremental page backup */ + BACKUP_MODE_DIFF_PTRACK, /* incremental page backup with ptrack system */ + BACKUP_MODE_DIFF_DELTA, /* incremental page backup with lsn comparison */ + BACKUP_MODE_FULL /* full backup */ +} BackupMode; + +typedef enum ProbackupSubcmd +{ + NO_CMD = 0, + INIT_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + DELETE_CMD, + MERGE_CMD, + SHOW_CMD, + SET_CONFIG_CMD, + SHOW_CONFIG_CMD +} ProbackupSubcmd; + +typedef enum ShowFormat +{ + SHOW_PLAIN, + SHOW_JSON +} ShowFormat; + + +/* special values of pgBackup fields */ +#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ +#define BYTES_INVALID (-1) +#define BLOCKNUM_INVALID (-1) + +typedef struct pgBackupConfig +{ + uint64 system_identifier; + uint32 xlog_seg_size; + + char *pgdata; + const char *pgdatabase; + const char *pghost; + const char *pgport; + const char *pguser; + + const char *master_host; + const char *master_port; + const char *master_db; + const char *master_user; + int replica_timeout; + + int archive_timeout; + + int log_level_console; + int log_level_file; + char *log_filename; + char *error_log_filename; + char *log_directory; + int log_rotation_size; + int log_rotation_age; + + uint32 retention_redundancy; + uint32 retention_window; + + CompressAlg compress_alg; + int compress_level; +} pgBackupConfig; + + +/* Information about single backup stored in backup.conf */ + + +typedef struct pgBackup pgBackup; + +struct pgBackup +{ + BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/ + time_t backup_id; /* Identifier of the backup. + * Currently it's the same as start_time */ + BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ + TimeLineID tli; /* timeline of start and stop baskup lsns */ + XLogRecPtr start_lsn; /* backup's starting transaction log location */ + XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ + time_t start_time; /* since this moment backup has status + * BACKUP_STATUS_RUNNING */ + time_t end_time; /* the moment when backup was finished, or the moment + * when we realized that backup is broken */ + time_t recovery_time; /* Earliest moment for which you can restore + * the state of the database cluster using + * this backup */ + TransactionId recovery_xid; /* Earliest xid for which you can restore + * the state of the database cluster using + * this backup */ + /* + * Amount of raw data. For a full backup, this is the total amount of + * data while for a differential backup this is just the difference + * of data taken. + * BYTES_INVALID means nothing was backed up. + */ + int64 data_bytes; + /* Size of WAL files in archive needed to restore this backup */ + int64 wal_bytes; + + CompressAlg compress_alg; + int compress_level; + + /* Fields needed for compatibility check */ + uint32 block_size; + uint32 wal_block_size; + uint32 checksum_version; + + char program_version[100]; + char server_version[100]; + + bool stream; /* Was this backup taken in stream mode? + * i.e. does it include all needed WAL files? */ + bool from_replica; /* Was this backup taken from replica */ + time_t parent_backup; /* Identifier of the previous backup. + * Which is basic backup for this + * incremental backup. */ + pgBackup *parent_backup_link; + char *primary_conninfo; /* Connection parameters of the backup + * in the format suitable for recovery.conf */ +}; + +/* Recovery target for restore and validate subcommands */ +typedef struct pgRecoveryTarget +{ + bool time_specified; + time_t recovery_target_time; + /* add one more field in order to avoid deparsing recovery_target_time back */ + const char *target_time_string; + bool xid_specified; + TransactionId recovery_target_xid; + /* add one more field in order to avoid deparsing recovery_target_xid back */ + const char *target_xid_string; + bool lsn_specified; + XLogRecPtr recovery_target_lsn; + /* add one more field in order to avoid deparsing recovery_target_lsn back */ + const char *target_lsn_string; + TimeLineID recovery_target_tli; + bool recovery_target_inclusive; + bool inclusive_specified; + bool recovery_target_immediate; + const char *recovery_target_name; + const char *recovery_target_action; + bool restore_no_validate; +} pgRecoveryTarget; + +/* Union to ease operations on relation pages */ +typedef union DataPage +{ + PageHeaderData page_data; + char data[BLCKSZ]; +} DataPage; + +typedef struct +{ + const char *from_root; + const char *to_root; + + parray *files_list; + parray *prev_filelist; + XLogRecPtr prev_start_lsn; + + PGconn *backup_conn; + PGcancel *cancel_conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} backup_files_arg; + +/* + * return pointer that exceeds the length of prefix from character string. + * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz". + */ +#define GetRelativePath(str, prefix) \ + ((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1) + +/* + * Return timeline, xlog ID and record offset from an LSN of the type + * 0/B000188, usual result from pg_stop_backup() and friends. + */ +#define XLogDataFromLSN(data, xlogid, xrecoff) \ + sscanf(data, "%X/%X", xlogid, xrecoff) + +#define IsCompressedXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) + +#if PG_VERSION_NUM >= 110000 +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) +#else +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo) +#endif + +/* directory options */ +extern char *backup_path; +extern char backup_instance_path[MAXPGPATH]; +extern char *pgdata; +extern char arclog_path[MAXPGPATH]; + +/* common options */ +extern int num_threads; +extern bool stream_wal; +extern bool progress; +#if PG_VERSION_NUM >= 100000 +/* In pre-10 'replication_slot' is defined in receivelog.h */ +extern char *replication_slot; +#endif + +/* backup options */ +extern bool smooth_checkpoint; +#define ARCHIVE_TIMEOUT_DEFAULT 300 +extern uint32 archive_timeout; +extern bool is_remote_backup; +extern const char *master_db; +extern const char *master_host; +extern const char *master_port; +extern const char *master_user; +#define REPLICA_TIMEOUT_DEFAULT 300 +extern uint32 replica_timeout; + +extern bool is_ptrack_support; +extern bool is_checksum_enabled; +extern bool exclusive_backup; + +/* restore options */ +extern bool restore_as_replica; + +/* delete options */ +extern bool delete_wal; +extern bool delete_expired; +extern bool apply_to_all; +extern bool force_delete; + +/* retention options. 0 disables the option */ +#define RETENTION_REDUNDANCY_DEFAULT 0 +#define RETENTION_WINDOW_DEFAULT 0 + +extern uint32 retention_redundancy; +extern uint32 retention_window; + +/* compression options */ +extern CompressAlg compress_alg; +extern int compress_level; +extern bool compress_shortcut; + +#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS +#define COMPRESS_LEVEL_DEFAULT 1 + +extern CompressAlg parse_compress_alg(const char *arg); +extern const char* deparse_compress_alg(int alg); +/* other options */ +extern char *instance_name; +extern uint64 system_identifier; +extern uint32 xlog_seg_size; + +/* show options */ +extern ShowFormat show_format; + +/* current settings */ +extern pgBackup current; +extern ProbackupSubcmd backup_subcmd; + +/* in dir.c */ +/* exclude directory list for $PGDATA file listing */ +extern const char *pgdata_exclude_dir[]; + +/* in backup.c */ +extern int do_backup(time_t start_time); +extern BackupMode parse_backup_mode(const char *value); +extern const char *deparse_backup_mode(BackupMode mode); +extern void process_block_change(ForkNumber forknum, RelFileNode rnode, + BlockNumber blkno); + +extern char *pg_ptrack_get_block(backup_files_arg *arguments, + Oid dbOid, Oid tblsOid, Oid relOid, + BlockNumber blknum, + size_t *result_size); +/* in restore.c */ +extern int do_restore_or_validate(time_t target_backup_id, + pgRecoveryTarget *rt, + bool is_restore); +extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); +extern bool satisfy_recovery_target(const pgBackup *backup, + const pgRecoveryTarget *rt); +extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); +extern pgRecoveryTarget *parseRecoveryTargetOptions( + const char *target_time, const char *target_xid, + const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, + bool target_immediate, const char *target_name, + const char *target_action, bool restore_no_validate); + +/* in merge.c */ +extern void do_merge(time_t backup_id); + +/* in init.c */ +extern int do_init(void); +extern int do_add_instance(void); + +/* in archive.c */ +extern int do_archive_push(char *wal_file_path, char *wal_file_name, + bool overwrite); +extern int do_archive_get(char *wal_file_path, char *wal_file_name); + + +/* in configure.c */ +extern int do_configure(bool show_only); +extern void pgBackupConfigInit(pgBackupConfig *config); +extern void writeBackupCatalogConfig(FILE *out, pgBackupConfig *config); +extern void writeBackupCatalogConfigFile(pgBackupConfig *config); +extern pgBackupConfig* readBackupCatalogConfigFile(void); + +extern uint32 get_config_xlog_seg_size(void); + +/* in show.c */ +extern int do_show(time_t requested_backup_id); + +/* in delete.c */ +extern int do_delete(time_t backup_id); +extern int do_retention_purge(void); +extern int do_delete_instance(void); + +/* in fetch.c */ +extern char *slurpFile(const char *datadir, + const char *path, + size_t *filesize, + bool safe); +extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); + +/* in help.c */ +extern void help_pg_probackup(void); +extern void help_command(char *command); + +/* in validate.c */ +extern void pgBackupValidate(pgBackup* backup); +extern int do_validate_all(void); + +/* in catalog.c */ +extern pgBackup *read_backup(time_t timestamp); +extern const char *pgBackupGetBackupMode(pgBackup *backup); + +extern parray *catalog_get_backup_list(time_t requested_backup_id); +extern pgBackup *catalog_get_last_data_backup(parray *backup_list, + TimeLineID tli); +extern void catalog_lock(void); +extern void pgBackupWriteControl(FILE *out, pgBackup *backup); +extern void pgBackupWriteBackupControlFile(pgBackup *backup); +extern void pgBackupWriteFileList(pgBackup *backup, parray *files, + const char *root); + +extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); +extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2); +extern int pgBackupCreateDir(pgBackup *backup); +extern void pgBackupInit(pgBackup *backup); +extern void pgBackupCopy(pgBackup *dst, pgBackup *src); +extern void pgBackupFree(void *backup); +extern int pgBackupCompareId(const void *f1, const void *f2); +extern int pgBackupCompareIdDesc(const void *f1, const void *f2); + +extern pgBackup* find_parent_backup(pgBackup *current_backup); + +/* in dir.c */ +extern void dir_list_file(parray *files, const char *root, bool exclude, + bool omit_symlink, bool add_root); +extern void create_data_directories(const char *data_dir, + const char *backup_dir, + bool extract_tablespaces); + +extern void read_tablespace_map(parray *files, const char *backup_dir); +extern void opt_tablespace_map(pgut_option *opt, const char *arg); +extern void check_tablespace_mapping(pgBackup *backup); + +extern void print_file_list(FILE *out, const parray *files, const char *root); +extern parray *dir_read_file_list(const char *root, const char *file_txt); + +extern int dir_create_dir(const char *path, mode_t mode); +extern bool dir_is_empty(const char *path); + +extern bool fileExists(const char *path); +extern size_t pgFileSize(const char *path); + +extern pgFile *pgFileNew(const char *path, bool omit_symlink); +extern pgFile *pgFileInit(const char *path); +extern void pgFileDelete(pgFile *file); +extern void pgFileFree(void *file); +extern pg_crc32 pgFileGetCRC(const char *file_path); +extern int pgFileComparePath(const void *f1, const void *f2); +extern int pgFileComparePathDesc(const void *f1, const void *f2); +extern int pgFileCompareLinked(const void *f1, const void *f2); +extern int pgFileCompareSize(const void *f1, const void *f2); + +/* in data.c */ +extern bool backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, + BackupMode backup_mode, + CompressAlg calg, int clevel); +extern void restore_data_file(const char *to_path, + pgFile *file, bool allow_truncate, + bool write_header); +extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern void move_file(const char *from_root, const char *to_root, pgFile *file); +extern void push_wal_file(const char *from_path, const char *to_path, + bool is_compress, bool overwrite); +extern void get_wal_file(const char *from_path, const char *to_path); + +extern bool calc_file_checksum(pgFile *file); + +/* parsexlog.c */ +extern void extractPageMap(const char *datadir, + TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, + bool prev_seg, parray *backup_files_list); +extern void validate_wal(pgBackup *backup, + const char *archivedir, + time_t target_time, + TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 seg_size); +extern bool read_recovery_info(const char *archivedir, TimeLineID tli, + uint32 seg_size, + XLogRecPtr start_lsn, XLogRecPtr stop_lsn, + time_t *recovery_time, + TransactionId *recovery_xid); +extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, + TimeLineID target_tli, uint32 seg_size); + +/* in util.c */ +extern TimeLineID get_current_timeline(bool safe); +extern void sanityChecks(void); +extern void time2iso(char *buf, size_t len, time_t time); +extern const char *status2str(BackupStatus status); +extern void remove_trailing_space(char *buf, int comment_mark); +extern void remove_not_digit(char *buf, size_t len, const char *str); +extern uint32 get_data_checksum_version(bool safe); +extern const char *base36enc(long unsigned int value); +extern char *base36enc_dup(long unsigned int value); +extern long unsigned int base36dec(const char *text); +extern uint64 get_system_identifier(char *pgdata); +extern uint64 get_remote_system_identifier(PGconn *conn); +extern uint32 get_xlog_seg_size(char *pgdata_path); +extern pg_time_t timestamptz_to_time_t(TimestampTz t); +extern int parse_server_version(char *server_version_str); + +/* in status.c */ +extern bool is_pg_running(void); + +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#define mkdir(dir,mode) _mkdir(dir) +#endif +#endif + +#endif /* PG_PROBACKUP_H */ diff --git a/src/restore.c b/src/restore.c new file mode 100644 index 00000000..3396b6f6 --- /dev/null +++ b/src/restore.c @@ -0,0 +1,919 @@ +/*------------------------------------------------------------------------- + * + * restore.c: restore DB cluster and archived WAL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "catalog/pg_control.h" +#include "utils/logger.h" +#include "utils/thread.h" + +typedef struct +{ + parray *files; + pgBackup *backup; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} restore_files_arg; + +static void restore_backup(pgBackup *backup); +static void create_recovery_conf(time_t backup_id, + pgRecoveryTarget *rt, + pgBackup *backup); +static void *restore_files(void *arg); +static void remove_deleted_files(pgBackup *backup); + + +/* + * Entry point of pg_probackup RESTORE and VALIDATE subcommands. + */ +int +do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, + bool is_restore) +{ + int i = 0; + parray *backups; + pgBackup *current_backup = NULL; + pgBackup *dest_backup = NULL; + pgBackup *base_full_backup = NULL; + pgBackup *corrupted_backup = NULL; + int dest_backup_index = 0; + int base_full_backup_index = 0; + int corrupted_backup_index = 0; + char *action = is_restore ? "Restore":"Validate"; + + if (is_restore) + { + if (pgdata == NULL) + elog(ERROR, + "required parameter not specified: PGDATA (-D, --pgdata)"); + /* Check if restore destination empty */ + if (!dir_is_empty(pgdata)) + elog(ERROR, "restore destination is not empty: \"%s\"", pgdata); + } + + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + + elog(LOG, "%s begin.", action); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find backup range we should restore or validate. */ + while ((i < parray_num(backups)) && !dest_backup) + { + current_backup = (pgBackup *) parray_get(backups, i); + i++; + + /* Skip all backups which started after target backup */ + if (target_backup_id && current_backup->start_time > target_backup_id) + continue; + + /* + * [PGPRO-1164] If BACKUP_ID is not provided for restore command, + * we must find the first valid(!) backup. + */ + + if (is_restore && + target_backup_id == INVALID_BACKUP_ID && + current_backup->status != BACKUP_STATUS_OK) + { + elog(WARNING, "Skipping backup %s, because it has non-valid status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + continue; + } + + /* + * We found target backup. Check its status and + * ensure that it satisfies recovery target. + */ + if ((target_backup_id == current_backup->start_time + || target_backup_id == INVALID_BACKUP_ID)) + { + + /* backup is not ok, + * but in case of CORRUPT, ORPHAN or DONE revalidation can be done, + * in other cases throw an error. + */ + if (current_backup->status != BACKUP_STATUS_OK) + { + if (current_backup->status == BACKUP_STATUS_DONE || + current_backup->status == BACKUP_STATUS_ORPHAN || + current_backup->status == BACKUP_STATUS_CORRUPT) + elog(WARNING, "Backup %s has status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + else + elog(ERROR, "Backup %s has status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + } + + if (rt->recovery_target_tli) + { + parray *timelines; + + elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); + /* Read timeline history files from archives */ + timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); + + if (!satisfy_timeline(timelines, current_backup)) + { + if (target_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "target backup %s does not satisfy target timeline", + base36enc(target_backup_id)); + else + /* Try to find another backup that satisfies target timeline */ + continue; + } + } + + if (!satisfy_recovery_target(current_backup, rt)) + { + if (target_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "target backup %s does not satisfy restore options", + base36enc(target_backup_id)); + else + /* Try to find another backup that satisfies target options */ + continue; + } + + /* + * Backup is fine and satisfies all recovery options. + * Save it as dest_backup + */ + dest_backup = current_backup; + dest_backup_index = i-1; + } + } + + if (dest_backup == NULL) + elog(ERROR, "Backup satisfying target options is not found."); + + /* If we already found dest_backup, look for full backup. */ + if (dest_backup) + { + base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* + * We have found full backup by link, + * now we need to walk the list to find its index. + * + * TODO I think we should rewrite it someday to use double linked list + * and avoid relying on sort order anymore. + */ + for (i = dest_backup_index; i < parray_num(backups); i++) + { + pgBackup * temp_backup = (pgBackup *) parray_get(backups, i); + if (temp_backup->start_time == base_full_backup->start_time) + { + base_full_backup_index = i; + break; + } + } + } + + if (base_full_backup == NULL) + elog(ERROR, "Full backup satisfying target options is not found."); + + /* + * Ensure that directories provided in tablespace mapping are valid + * i.e. empty or not exist. + */ + if (is_restore) + check_tablespace_mapping(dest_backup); + + if (!is_restore || !rt->restore_no_validate) + { + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); + + /* + * Validate backups from base_full_backup to dest_backup. + */ + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + pgBackupValidate(backup); + /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ + if (backup->status == BACKUP_STATUS_CORRUPT) + { + corrupted_backup = backup; + corrupted_backup_index = i; + break; + } + /* We do not validate WAL files of intermediate backups + * It`s done to speed up restore + */ + } + /* There is no point in wal validation + * if there is corrupted backup between base_backup and dest_backup + */ + if (!corrupted_backup) + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->recovery_target_time, + rt->recovery_target_xid, rt->recovery_target_lsn, + base_full_backup->tli, xlog_seg_size); + + /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (corrupted_backup) + { + for (i = corrupted_backup_index - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + /* Mark incremental OK backup as orphan */ + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + char *backup_id, + *corrupted_backup_id; + + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); + + backup_id = base36enc_dup(backup->start_time); + corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); + + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + backup_id, corrupted_backup_id); + + free(backup_id); + free(corrupted_backup_id); + } + } + } + } + + /* + * If dest backup is corrupted or was orphaned in previous check + * produce corresponding error message + */ + if (dest_backup->status == BACKUP_STATUS_OK) + { + if (rt->restore_no_validate) + elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + else + elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + } + else if (dest_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + else if (dest_backup->status == BACKUP_STATUS_ORPHAN) + elog(ERROR, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + else + elog(ERROR, "Backup %s has status: %s", + base36enc(dest_backup->start_time), status2str(dest_backup->status)); + + /* We ensured that all backups are valid, now restore if required */ + if (is_restore) + { + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (rt->lsn_specified && parse_server_version(backup->server_version) < 100000) + elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", + base36enc(dest_backup->start_time), dest_backup->server_version); + + restore_backup(backup); + } + + /* + * Delete files which are not in dest backup file list. Files which were + * deleted between previous and current backup are not in the list. + */ + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + remove_deleted_files(dest_backup); + + /* Create recovery.conf with given recovery target parameters */ + create_recovery_conf(target_backup_id, rt, dest_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(INFO, "%s of backup %s completed.", + action, base36enc(dest_backup->start_time)); + return 0; +} + +/* + * Restore one backup. + */ +void +restore_backup(pgBackup *backup) +{ + char timestamp[100]; + char this_backup_path[MAXPGPATH]; + char database_path[MAXPGPATH]; + char list_path[MAXPGPATH]; + parray *files; + int i; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + restore_files_arg *threads_args; + bool restore_isok = true; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); + + /* confirm block size compatibility */ + if (backup->block_size != BLCKSZ) + elog(ERROR, + "BLCKSZ(%d) is not compatible(%d expected)", + backup->block_size, BLCKSZ); + if (backup->wal_block_size != XLOG_BLCKSZ) + elog(ERROR, + "XLOG_BLCKSZ(%d) is not compatible(%d expected)", + backup->wal_block_size, XLOG_BLCKSZ); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + elog(LOG, "restoring database from backup %s", timestamp); + + /* + * Restore backup directories. + * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id + */ + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + create_data_directories(pgdata, this_backup_path, true); + + /* + * Get list of files which need to be restored. + */ + pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); + pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); + files = dir_read_file_list(database_path, list_path); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_clear_flag(&file->lock); + } + + /* Restore files into target directory */ + for (i = 0; i < num_threads; i++) + { + restore_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->backup = backup; + /* By default there are some error */ + threads_args[i].ret = 1; + + elog(LOG, "Start thread for num:%li", parray_num(files)); + + pthread_create(&threads[i], NULL, restore_files, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + restore_isok = false; + } + if (!restore_isok) + elog(ERROR, "Data files restoring failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); +} + +/* + * Delete files which are not in backup's file list from target pgdata. + * It is necessary to restore incremental backup correctly. + * Files which were deleted between previous and current backup + * are not in the backup's filelist. + */ +static void +remove_deleted_files(pgBackup *backup) +{ + parray *files; + parray *files_restored; + char filelist_path[MAXPGPATH]; + int i; + + pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); + /* Read backup's filelist using target database path as base path */ + files = dir_read_file_list(pgdata, filelist_path); + parray_qsort(files, pgFileComparePathDesc); + + /* Get list of files actually existing in target database */ + files_restored = parray_new(); + dir_list_file(files_restored, pgdata, true, true, false); + /* To delete from leaf, sort in reversed order */ + parray_qsort(files_restored, pgFileComparePathDesc); + + for (i = 0; i < parray_num(files_restored); i++) + { + pgFile *file = (pgFile *) parray_get(files_restored, i); + + /* If the file is not in the file list, delete it */ + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); + } + } + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + parray_walk(files_restored, pgFileFree); + parray_free(files_restored); +} + +/* + * Restore files into $PGDATA. + */ +static void * +restore_files(void *arg) +{ + int i; + restore_files_arg *arguments = (restore_files_arg *)arg; + + for (i = 0; i < parray_num(arguments->files); i++) + { + char from_root[MAXPGPATH]; + char *rel_path; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + pgBackupGetPath(arguments->backup, from_root, + lengthof(from_root), DATABASE_DIR); + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during restore database"); + + rel_path = GetRelativePath(file->path,from_root); + + if (progress) + elog(LOG, "Progress: (%d/%lu). Process file %s ", + i + 1, (unsigned long) parray_num(arguments->files), rel_path); + + /* + * For PAGE and PTRACK backups skip files which haven't changed + * since previous backup and thus were not backed up. + * We cannot do the same when restoring DELTA backup because we need information + * about every file to correctly truncate them. + */ + if (file->write_size == BYTES_INVALID && + (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE + || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) + { + elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + { + elog(VERBOSE, "directory, skip"); + continue; + } + + /* Do not restore tablespace_map file */ + if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, rel_path)) + { + elog(VERBOSE, "skip tablespace_map"); + continue; + } + + /* + * restore the file. + * We treat datafiles separately, cause they were backed up block by + * block and have BackupPageHeader meta information, so we cannot just + * copy the file from backup. + */ + elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", + file->path, file->is_datafile?1:0, file->is_cfs?1:0); + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, pgdata, + file->path + strlen(from_root) + 1); + restore_data_file(to_path, file, + arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + } + else + copy_file(from_root, pgdata, file); + + /* print size of restored file */ + if (file->write_size != BYTES_INVALID) + elog(LOG, "Restored file %s : " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files restoring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* Create recovery.conf with given recovery target parameters */ +static void +create_recovery_conf(time_t backup_id, + pgRecoveryTarget *rt, + pgBackup *backup) +{ + char path[MAXPGPATH]; + FILE *fp; + bool need_restore_conf = false; + + if (!backup->stream + || (rt->time_specified || rt->xid_specified)) + need_restore_conf = true; + + /* No need to generate recovery.conf at all. */ + if (!(need_restore_conf || restore_as_replica)) + return; + + elog(LOG, "----------------------------------------"); + elog(LOG, "creating recovery.conf"); + + snprintf(path, lengthof(path), "%s/recovery.conf", pgdata); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, + strerror(errno)); + + fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", + PROGRAM_VERSION); + + if (need_restore_conf) + { + + fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " + "--wal-file-path %%p --wal-file-name %%f'\n", + PROGRAM_NAME, backup_path, instance_name); + + /* + * We've already checked that only one of the four following mutually + * exclusive options is specified, so the order of calls is insignificant. + */ + if (rt->recovery_target_name) + fprintf(fp, "recovery_target_name = '%s'\n", rt->recovery_target_name); + + if (rt->time_specified) + fprintf(fp, "recovery_target_time = '%s'\n", rt->target_time_string); + + if (rt->xid_specified) + fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); + + if (rt->recovery_target_lsn) + fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); + + if (rt->recovery_target_immediate) + fprintf(fp, "recovery_target = 'immediate'\n"); + + if (rt->inclusive_specified) + fprintf(fp, "recovery_target_inclusive = '%s'\n", + rt->recovery_target_inclusive?"true":"false"); + + if (rt->recovery_target_tli) + fprintf(fp, "recovery_target_timeline = '%u'\n", rt->recovery_target_tli); + + if (rt->recovery_target_action) + fprintf(fp, "recovery_target_action = '%s'\n", rt->recovery_target_action); + } + + if (restore_as_replica) + { + fprintf(fp, "standby_mode = 'on'\n"); + + if (backup->primary_conninfo) + fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + } + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write recovery.conf \"%s\": %s", path, + strerror(errno)); +} + +/* + * Try to read a timeline's history file. + * + * If successful, return the list of component TLIs (the ancestor + * timelines followed by target timeline). If we cannot find the history file, + * assume that the timeline has no parents, and return a list of just the + * specified timeline ID. + * based on readTimeLineHistory() in timeline.c + */ +parray * +readTimeLineHistory_probackup(TimeLineID targetTLI) +{ + parray *result; + char path[MAXPGPATH]; + char fline[MAXPGPATH]; + FILE *fd = NULL; + TimeLineHistoryEntry *entry; + TimeLineHistoryEntry *last_timeline = NULL; + + /* Look for timeline history file in archlog_path */ + snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, + targetTLI); + + /* Timeline 1 does not have a history file */ + if (targetTLI != 1) + { + fd = fopen(path, "rt"); + if (fd == NULL) + { + if (errno != ENOENT) + elog(ERROR, "could not open file \"%s\": %s", path, + strerror(errno)); + + /* There is no history file for target timeline */ + elog(ERROR, "recovery target timeline %u does not exist", + targetTLI); + } + } + + result = parray_new(); + + /* + * Parse the file... + */ + while (fd && fgets(fline, sizeof(fline), fd) != NULL) + { + char *ptr; + TimeLineID tli; + uint32 switchpoint_hi; + uint32 switchpoint_lo; + int nfields; + + for (ptr = fline; *ptr; ptr++) + { + if (!isspace((unsigned char) *ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + nfields = sscanf(fline, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); + + if (nfields < 1) + { + /* expect a numeric timeline ID as first field of line */ + elog(ERROR, + "syntax error in history file: %s. Expected a numeric timeline ID.", + fline); + } + if (nfields != 3) + elog(ERROR, + "syntax error in history file: %s. Expected a transaction log switchpoint location.", + fline); + + if (last_timeline && tli <= last_timeline->tli) + elog(ERROR, + "Timeline IDs must be in increasing sequence."); + + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = tli; + entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; + + last_timeline = entry; + /* Build list with newest item first */ + parray_insert(result, 0, entry); + + /* we ignore the remainder of each line */ + } + + if (fd) + fclose(fd); + + if (last_timeline && targetTLI <= last_timeline->tli) + elog(ERROR, "Timeline IDs must be less than child timeline's ID."); + + /* append target timeline */ + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = targetTLI; + /* LSN in target timeline is valid */ + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; + parray_insert(result, 0, entry); + + return result; +} + +bool +satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) +{ + if (rt->xid_specified) + return backup->recovery_xid <= rt->recovery_target_xid; + + if (rt->time_specified) + return backup->recovery_time <= rt->recovery_target_time; + + if (rt->lsn_specified) + return backup->stop_lsn <= rt->recovery_target_lsn; + + return true; +} + +bool +satisfy_timeline(const parray *timelines, const pgBackup *backup) +{ + int i; + + for (i = 0; i < parray_num(timelines); i++) + { + TimeLineHistoryEntry *timeline; + + timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + if (backup->tli == timeline->tli && + backup->stop_lsn < timeline->end) + return true; + } + return false; +} +/* + * Get recovery options in the string format, parse them + * and fill up the pgRecoveryTarget structure. + */ +pgRecoveryTarget * +parseRecoveryTargetOptions(const char *target_time, + const char *target_xid, + const char *target_inclusive, + TimeLineID target_tli, + const char *target_lsn, + bool target_immediate, + const char *target_name, + const char *target_action, + bool restore_no_validate) +{ + time_t dummy_time; + TransactionId dummy_xid; + bool dummy_bool; + XLogRecPtr dummy_lsn; + /* + * count the number of the mutually exclusive options which may specify + * recovery target. If final value > 1, throw an error. + */ + int recovery_target_specified = 0; + pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); + + /* fill all options with default values */ + rt->time_specified = false; + rt->xid_specified = false; + rt->inclusive_specified = false; + rt->lsn_specified = false; + rt->recovery_target_time = 0; + rt->recovery_target_xid = 0; + rt->recovery_target_lsn = InvalidXLogRecPtr; + rt->target_time_string = NULL; + rt->target_xid_string = NULL; + rt->target_lsn_string = NULL; + rt->recovery_target_inclusive = false; + rt->recovery_target_tli = 0; + rt->recovery_target_immediate = false; + rt->recovery_target_name = NULL; + rt->recovery_target_action = NULL; + rt->restore_no_validate = false; + + /* parse given options */ + if (target_time) + { + recovery_target_specified++; + rt->time_specified = true; + rt->target_time_string = target_time; + + if (parse_time(target_time, &dummy_time, false)) + rt->recovery_target_time = dummy_time; + else + elog(ERROR, "Invalid value of --time option %s", target_time); + } + + if (target_xid) + { + recovery_target_specified++; + rt->xid_specified = true; + rt->target_xid_string = target_xid; + +#ifdef PGPRO_EE + if (parse_uint64(target_xid, &dummy_xid, 0)) +#else + if (parse_uint32(target_xid, &dummy_xid, 0)) +#endif + rt->recovery_target_xid = dummy_xid; + else + elog(ERROR, "Invalid value of --xid option %s", target_xid); + } + + if (target_lsn) + { + recovery_target_specified++; + rt->lsn_specified = true; + rt->target_lsn_string = target_lsn; + if (parse_lsn(target_lsn, &dummy_lsn)) + rt->recovery_target_lsn = dummy_lsn; + else + elog(ERROR, "Invalid value of --lsn option %s", target_lsn); + } + + if (target_inclusive) + { + rt->inclusive_specified = true; + if (parse_bool(target_inclusive, &dummy_bool)) + rt->recovery_target_inclusive = dummy_bool; + else + elog(ERROR, "Invalid value of --inclusive option %s", target_inclusive); + } + + rt->recovery_target_tli = target_tli; + if (target_immediate) + { + recovery_target_specified++; + rt->recovery_target_immediate = target_immediate; + } + + if (restore_no_validate) + { + rt->restore_no_validate = restore_no_validate; + } + + if (target_name) + { + recovery_target_specified++; + rt->recovery_target_name = target_name; + } + + if (target_action) + { + rt->recovery_target_action = target_action; + + if ((strcmp(target_action, "pause") != 0) + && (strcmp(target_action, "promote") != 0) + && (strcmp(target_action, "shutdown") != 0)) + elog(ERROR, "Invalid value of --recovery-target-action option %s", target_action); + } + else + { + /* Default recovery target action is pause */ + rt->recovery_target_action = "pause"; + } + + /* More than one mutually exclusive option was defined. */ + if (recovery_target_specified > 1) + elog(ERROR, "At most one of --immediate, --target-name, --time, --xid, or --lsn can be used"); + + /* If none of the options is defined, '--inclusive' option is meaningless */ + if (!(rt->xid_specified || rt->time_specified || rt->lsn_specified) && rt->recovery_target_inclusive) + elog(ERROR, "--inclusive option applies when either --time or --xid is specified"); + + return rt; +} diff --git a/src/show.c b/src/show.c new file mode 100644 index 00000000..f240ce93 --- /dev/null +++ b/src/show.c @@ -0,0 +1,500 @@ +/*------------------------------------------------------------------------- + * + * show.c: show backup information. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "pqexpbuffer.h" + +#include "utils/json.h" + + +static void show_instance_start(void); +static void show_instance_end(void); +static void show_instance(time_t requested_backup_id, bool show_name); +static int show_backup(time_t requested_backup_id); + +static void show_instance_plain(parray *backup_list, bool show_name); +static void show_instance_json(parray *backup_list); + +static PQExpBufferData show_buf; +static bool first_instance = true; +static int32 json_level = 0; + +int +do_show(time_t requested_backup_id) +{ + if (instance_name == NULL && + requested_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "You must specify --instance to use --backup_id option"); + + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "Cannot open directory \"%s\": %s", + path, strerror(errno)); + + show_instance_start(); + + while (errno = 0, (dent = readdir(dir)) != NULL) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", + child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + + show_instance(INVALID_BACKUP_ID, true); + } + + if (errno) + elog(ERROR, "Cannot read directory \"%s\": %s", + path, strerror(errno)); + + if (closedir(dir)) + elog(ERROR, "Cannot close directory \"%s\": %s", + path, strerror(errno)); + + show_instance_end(); + + return 0; + } + else if (requested_backup_id == INVALID_BACKUP_ID || + show_format == SHOW_JSON) + { + show_instance_start(); + show_instance(requested_backup_id, false); + show_instance_end(); + + return 0; + } + else + return show_backup(requested_backup_id); +} + +static void +pretty_size(int64 size, char *buf, size_t len) +{ + int exp = 0; + + /* minus means the size is invalid */ + if (size < 0) + { + strncpy(buf, "----", len); + return; + } + + /* determine postfix */ + while (size > 9999) + { + ++exp; + size /= 1000; + } + + switch (exp) + { + case 0: + snprintf(buf, len, "%dB", (int) size); + break; + case 1: + snprintf(buf, len, "%dkB", (int) size); + break; + case 2: + snprintf(buf, len, "%dMB", (int) size); + break; + case 3: + snprintf(buf, len, "%dGB", (int) size); + break; + case 4: + snprintf(buf, len, "%dTB", (int) size); + break; + case 5: + snprintf(buf, len, "%dPB", (int) size); + break; + default: + strncpy(buf, "***", len); + break; + } +} + +static TimeLineID +get_parent_tli(TimeLineID child_tli) +{ + TimeLineID result = 0; + char path[MAXPGPATH]; + char fline[MAXPGPATH]; + FILE *fd; + + /* Timeline 1 does not have a history file and parent timeline */ + if (child_tli == 1) + return 0; + + /* Search history file in archives */ + snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, + child_tli); + fd = fopen(path, "rt"); + if (fd == NULL) + { + if (errno != ENOENT) + elog(ERROR, "could not open file \"%s\": %s", path, + strerror(errno)); + + /* Did not find history file, do not raise the error */ + return 0; + } + + /* + * Parse the file... + */ + while (fgets(fline, sizeof(fline), fd) != NULL) + { + /* skip leading whitespace and check for # comment */ + char *ptr; + char *endptr; + + for (ptr = fline; *ptr; ptr++) + { + if (!IsSpace(*ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + /* expect a numeric timeline ID as first field of line */ + result = (TimeLineID) strtoul(ptr, &endptr, 0); + if (endptr == ptr) + elog(ERROR, + "syntax error(timeline ID) in history file: %s", + fline); + } + + fclose(fd); + + /* TLI of the last line is parent TLI */ + return result; +} + +/* + * Initialize instance visualization. + */ +static void +show_instance_start(void) +{ + initPQExpBuffer(&show_buf); + + if (show_format == SHOW_PLAIN) + return; + + first_instance = true; + json_level = 0; + + appendPQExpBufferChar(&show_buf, '['); + json_level++; +} + +/* + * Finalize instance visualization. + */ +static void +show_instance_end(void) +{ + if (show_format == SHOW_JSON) + appendPQExpBufferStr(&show_buf, "\n]\n"); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show brief meta information about all backups in the backup instance. + */ +static void +show_instance(time_t requested_backup_id, bool show_name) +{ + parray *backup_list; + + backup_list = catalog_get_backup_list(requested_backup_id); + + if (show_format == SHOW_PLAIN) + show_instance_plain(backup_list, show_name); + else if (show_format == SHOW_JSON) + show_instance_json(backup_list); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* + * Show detailed meta information about specified backup. + */ +static int +show_backup(time_t requested_backup_id) +{ + pgBackup *backup; + + backup = read_backup(requested_backup_id); + if (backup == NULL) + { + elog(INFO, "Requested backup \"%s\" is not found.", + /* We do not need free base36enc's result, we exit anyway */ + base36enc(requested_backup_id)); + /* This is not error */ + return 0; + } + + if (show_format == SHOW_PLAIN) + pgBackupWriteControl(stdout, backup); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + pgBackupFree(backup); + + return 0; +} + +/* + * Plain output. + */ + +/* + * Show instance backups in plain format. + */ +static void +show_instance_plain(parray *backup_list, bool show_name) +{ + int i; + + if (show_name) + printfPQExpBuffer(&show_buf, "\nBACKUP INSTANCE '%s'\n", instance_name); + + /* if you add new fields here, fix the header */ + /* show header */ + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + appendPQExpBufferStr(&show_buf, + " Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n"); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char duration[20] = "----"; + char data_bytes_str[10] = "----"; + + if (backup->recovery_time != (time_t) 0) + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + if (backup->end_time != (time_t) 0) + snprintf(duration, lengthof(duration), "%.*lfs", 0, + difftime(backup->end_time, backup->start_time)); + + /* + * Calculate Data field, in the case of full backup this shows the + * total amount of data. For an differential backup, this size is only + * the difference of data accumulated. + */ + pretty_size(backup->data_bytes, data_bytes_str, + lengthof(data_bytes_str)); + + /* Get parent timeline before printing */ + parent_tli = get_parent_tli(backup->tli); + + appendPQExpBuffer(&show_buf, + " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, + (backup->server_version[0] ? backup->server_version : "----"), + base36enc(backup->start_time), + timestamp, + pgBackupGetBackupMode(backup), + backup->stream ? "STREAM": "ARCHIVE", + backup->tli, + parent_tli, + duration, + data_bytes_str, + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn, + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn, + status2str(backup->status)); + } +} + +/* + * Json output. + */ + +/* + * Show instance backups in json format. + */ +static void +show_instance_json(parray *backup_list) +{ + int i; + PQExpBuffer buf = &show_buf; + + if (!first_instance) + appendPQExpBufferChar(buf, ','); + + /* Begin of instance object */ + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "instance", instance_name, json_level, false); + json_add_key(buf, "backups", json_level, true); + + /* + * List backups. + */ + json_add(buf, JT_BEGIN_ARRAY, &json_level); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char lsn[20]; + + if (i != 0) + appendPQExpBufferChar(buf, ','); + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "id", base36enc(backup->start_time), json_level, + false); + + if (backup->parent_backup != 0) + json_add_value(buf, "parent-backup-id", + base36enc(backup->parent_backup), json_level, true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_level, true); + + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", + json_level, true); + + json_add_value(buf, "compress-alg", + deparse_compress_alg(backup->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", backup->compress_level); + + json_add_value(buf, "from-replica", + backup->from_replica ? "true" : "false", json_level, + true); + + json_add_key(buf, "block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->block_size); + + json_add_key(buf, "xlog-block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->wal_block_size); + + json_add_key(buf, "checksum-version", json_level, true); + appendPQExpBuffer(buf, "%u", backup->checksum_version); + + json_add_value(buf, "program-version", backup->program_version, + json_level, true); + json_add_value(buf, "server-version", backup->server_version, + json_level, true); + + json_add_key(buf, "current-tli", json_level, true); + appendPQExpBuffer(buf, "%d", backup->tli); + + json_add_key(buf, "parent-tli", json_level, true); + parent_tli = get_parent_tli(backup->tli); + appendPQExpBuffer(buf, "%u", parent_tli); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); + json_add_value(buf, "start-lsn", lsn, json_level, true); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); + json_add_value(buf, "stop-lsn", lsn, json_level, true); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + json_add_value(buf, "start-time", timestamp, json_level, true); + + if (backup->end_time) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, json_level, true); + } + + json_add_key(buf, "recovery-xid", json_level, true); + appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, json_level, true); + } + + if (backup->data_bytes != BYTES_INVALID) + { + json_add_key(buf, "data-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + } + + if (backup->wal_bytes != BYTES_INVALID) + { + json_add_key(buf, "wal-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); + } + + if (backup->primary_conninfo) + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, + json_level, true); + + json_add_value(buf, "status", status2str(backup->status), json_level, + true); + + json_add(buf, JT_END_OBJECT, &json_level); + } + + /* End of backups */ + json_add(buf, JT_END_ARRAY, &json_level); + + /* End of instance object */ + json_add(buf, JT_END_OBJECT, &json_level); + + first_instance = false; +} diff --git a/src/status.c b/src/status.c new file mode 100644 index 00000000..155a07f4 --- /dev/null +++ b/src/status.c @@ -0,0 +1,118 @@ +/*------------------------------------------------------------------------- + * + * status.c + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * + * Monitor status of a PostgreSQL server. + * + *------------------------------------------------------------------------- + */ + + +#include "postgres_fe.h" + +#include +#include +#include + +#include "pg_probackup.h" + +/* PID can be negative for standalone backend */ +typedef long pgpid_t; + +static pgpid_t get_pgpid(void); +static bool postmaster_is_alive(pid_t pid); + +/* + * get_pgpid + * + * Get PID of postmaster, by scanning postmaster.pid. + */ +static pgpid_t +get_pgpid(void) +{ + FILE *pidf; + long pid; + char pid_file[MAXPGPATH]; + + snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); + + pidf = fopen(pid_file, PG_BINARY_R); + if (pidf == NULL) + { + /* No pid file, not an error on startup */ + if (errno == ENOENT) + return 0; + else + { + elog(ERROR, "could not open PID file \"%s\": %s", + pid_file, strerror(errno)); + } + } + if (fscanf(pidf, "%ld", &pid) != 1) + { + /* Is the file empty? */ + if (ftell(pidf) == 0 && feof(pidf)) + elog(ERROR, "the PID file \"%s\" is empty", + pid_file); + else + elog(ERROR, "invalid data in PID file \"%s\"\n", + pid_file); + } + fclose(pidf); + return (pgpid_t) pid; +} + +/* + * postmaster_is_alive + * + * Check whether postmaster is alive or not. + */ +static bool +postmaster_is_alive(pid_t pid) +{ + /* + * Test to see if the process is still there. Note that we do not + * consider an EPERM failure to mean that the process is still there; + * EPERM must mean that the given PID belongs to some other userid, and + * considering the permissions on $PGDATA, that means it's not the + * postmaster we are after. + * + * Don't believe that our own PID or parent shell's PID is the postmaster, + * either. (Windows hasn't got getppid(), though.) + */ + if (pid == getpid()) + return false; +#ifndef WIN32 + if (pid == getppid()) + return false; +#endif + if (kill(pid, 0) == 0) + return true; + return false; +} + +/* + * is_pg_running + * + * + */ +bool +is_pg_running(void) +{ + pgpid_t pid; + + pid = get_pgpid(); + + /* 0 means no pid file */ + if (pid == 0) + return false; + + /* Case of a standalone backend */ + if (pid < 0) + pid = -pid; + + /* Check if postmaster is alive */ + return postmaster_is_alive((pid_t) pid); +} diff --git a/src/util.c b/src/util.c new file mode 100644 index 00000000..82814d11 --- /dev/null +++ b/src/util.c @@ -0,0 +1,349 @@ +/*------------------------------------------------------------------------- + * + * util.c: log messages to log file or stderr, and misc code. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include + +#include "storage/bufpage.h" +#if PG_VERSION_NUM >= 110000 +#include "streamutil.h" +#endif + +const char * +base36enc(long unsigned int value) +{ + const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ + static char buffer[14]; + unsigned int offset = sizeof(buffer); + + buffer[--offset] = '\0'; + do { + buffer[--offset] = base36[value % 36]; + } while (value /= 36); + + return &buffer[offset]; +} + +/* + * Same as base36enc(), but the result must be released by the user. + */ +char * +base36enc_dup(long unsigned int value) +{ + const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ + char buffer[14]; + unsigned int offset = sizeof(buffer); + + buffer[--offset] = '\0'; + do { + buffer[--offset] = base36[value % 36]; + } while (value /= 36); + + return strdup(&buffer[offset]); +} + +long unsigned int +base36dec(const char *text) +{ + return strtoul(text, NULL, 36); +} + +static void +checkControlFile(ControlFileData *ControlFile) +{ + pg_crc32c crc; + + /* Calculate CRC */ + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *) ControlFile, offsetof(ControlFileData, crc)); + FIN_CRC32C(crc); + + /* Then compare it */ + if (!EQ_CRC32C(crc, ControlFile->crc)) + elog(ERROR, "Calculated CRC checksum does not match value stored in file.\n" + "Either the file is corrupt, or it has a different layout than this program\n" + "is expecting. The results below are untrustworthy."); + + if (ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0) + elog(ERROR, "possible byte ordering mismatch\n" + "The byte ordering used to store the pg_control file might not match the one\n" + "used by this program. In that case the results below would be incorrect, and\n" + "the PostgreSQL installation would be incompatible with this data directory."); +} + +/* + * Verify control file contents in the buffer src, and copy it to *ControlFile. + */ +static void +digestControlFile(ControlFileData *ControlFile, char *src, size_t size) +{ +#if PG_VERSION_NUM >= 100000 + int ControlFileSize = PG_CONTROL_FILE_SIZE; +#else + int ControlFileSize = PG_CONTROL_SIZE; +#endif + + if (size != ControlFileSize) + elog(ERROR, "unexpected control file size %d, expected %d", + (int) size, ControlFileSize); + + memcpy(ControlFile, src, sizeof(ControlFileData)); + + /* Additional checks on control file */ + checkControlFile(ControlFile); +} + +/* + * Utility shared by backup and restore to fetch the current timeline + * used by a node. + */ +TimeLineID +get_current_timeline(bool safe) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + if (safe && buffer == NULL) + return 0; + + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.checkPointCopy.ThisTimeLineID; +} + +uint64 +get_system_identifier(char *pgdata_path) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.system_identifier; +} + +uint64 +get_remote_system_identifier(PGconn *conn) +{ +#if PG_VERSION_NUM >= 90600 + PGresult *res; + uint64 system_id_conn; + char *val; + + res = pgut_execute(conn, + "SELECT system_identifier FROM pg_catalog.pg_control_system()", + 0, NULL); + val = PQgetvalue(res, 0, 0); + if (!parse_uint64(val, &system_id_conn, 0)) + { + PQclear(res); + elog(ERROR, "%s is not system_identifier", val); + } + PQclear(res); + + return system_id_conn; +#else + char *buffer; + size_t size; + ControlFileData ControlFile; + + buffer = fetchFile(conn, "global/pg_control", &size); + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.system_identifier; +#endif +} + +uint32 +get_xlog_seg_size(char *pgdata_path) +{ +#if PG_VERSION_NUM >= 110000 + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.xlog_seg_size; +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + +uint32 +get_data_checksum_version(bool safe) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.data_checksum_version; +} + + +/* + * Convert time_t value to ISO-8601 format string. Always set timezone offset. + */ +void +time2iso(char *buf, size_t len, time_t time) +{ + struct tm *ptm = gmtime(&time); + time_t gmt = mktime(ptm); + time_t offset; + char *ptr = buf; + + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); + + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", + (offset >= 0) ? '+' : '-', + abs((int) offset) / SECS_PER_HOUR); + + if (abs((int) offset) % SECS_PER_HOUR != 0) + { + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", + abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); + } +} + +/* copied from timestamp.c */ +pg_time_t +timestamptz_to_time_t(TimestampTz t) +{ + pg_time_t result; + +#ifdef HAVE_INT64_TIMESTAMP + result = (pg_time_t) (t / USECS_PER_SEC + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); +#else + result = (pg_time_t) (t + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); +#endif + return result; +} + +/* Parse string representation of the server version */ +int +parse_server_version(char *server_version_str) +{ + int nfields; + int result = 0; + int major_version = 0; + int minor_version = 0; + + nfields = sscanf(server_version_str, "%d.%d", &major_version, &minor_version); + if (nfields == 2) + { + /* Server version lower than 10 */ + if (major_version > 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000 + minor_version * 100; + } + else if (nfields == 1) + { + if (major_version < 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000; + } + else + elog(ERROR, "Unknown server version format"); + + return result; +} + +const char * +status2str(BackupStatus status) +{ + static const char *statusName[] = + { + "UNKNOWN", + "OK", + "ERROR", + "RUNNING", + "MERGING", + "DELETING", + "DELETED", + "DONE", + "ORPHAN", + "CORRUPT" + }; + if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status) + return "UNKNOWN"; + + return statusName[status]; +} + +void +remove_trailing_space(char *buf, int comment_mark) +{ + int i; + char *last_char = NULL; + + for (i = 0; buf[i]; i++) + { + if (buf[i] == comment_mark || buf[i] == '\n' || buf[i] == '\r') + { + buf[i] = '\0'; + break; + } + } + for (i = 0; buf[i]; i++) + { + if (!isspace(buf[i])) + last_char = buf + i; + } + if (last_char != NULL) + *(last_char + 1) = '\0'; + +} + +void +remove_not_digit(char *buf, size_t len, const char *str) +{ + int i, j; + + for (i = 0, j = 0; str[i] && j < len; i++) + { + if (!isdigit(str[i])) + continue; + buf[j++] = str[i]; + } + buf[j] = '\0'; +} diff --git a/src/utils/json.c b/src/utils/json.c new file mode 100644 index 00000000..3afbe9e7 --- /dev/null +++ b/src/utils/json.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * json.c: - make json document. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "json.h" + +static void json_add_indent(PQExpBuffer buf, int32 level); +static void json_add_escaped(PQExpBuffer buf, const char *str); + +/* + * Start or end json token. Currently it is a json object or array. + * + * Function modifies level value and adds indent if it appropriate. + */ +void +json_add(PQExpBuffer buf, JsonToken type, int32 *level) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + *level += 1; + break; + case JT_END_ARRAY: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '{'); + *level += 1; + break; + case JT_END_OBJECT: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; + } +} + +/* + * Add json object's key. If it isn't first key we need to add a comma. + */ +void +json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf, level); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +/* + * Add json object's key and value. If it isn't first key we need to add a + * comma. + */ +void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma) +{ + json_add_key(buf, name, level, add_comma); + json_add_escaped(buf, value); +} + +static void +json_add_indent(PQExpBuffer buf, int32 level) +{ + uint16 i; + + if (level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < level; i++) + appendPQExpBufferStr(buf, " "); +} + +static void +json_add_escaped(PQExpBuffer buf, const char *str) +{ + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} diff --git a/src/utils/json.h b/src/utils/json.h new file mode 100644 index 00000000..cf5a7064 --- /dev/null +++ b/src/utils/json.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * json.h: - prototypes of json output functions. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_JSON_H +#define PROBACKUP_JSON_H + +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +/* + * Json document tokens. + */ +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_key(PQExpBuffer buf, const char *name, int32 level, + bool add_comma); +extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma); + +#endif /* PROBACKUP_JSON_H */ diff --git a/src/utils/logger.c b/src/utils/logger.c new file mode 100644 index 00000000..31669ed0 --- /dev/null +++ b/src/utils/logger.c @@ -0,0 +1,621 @@ +/*------------------------------------------------------------------------- + * + * logger.c: - log events into log file or stderr. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +#include "logger.h" +#include "pgut.h" +#include "pg_probackup.h" +#include "thread.h" + +/* Logger parameters */ + +int log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; +int log_level_file = LOG_LEVEL_FILE_DEFAULT; + +char *log_filename = NULL; +char *error_log_filename = NULL; +char *log_directory = NULL; +/* + * If log_path is empty logging is not initialized. + * We will log only into stderr + */ +char log_path[MAXPGPATH] = ""; + +/* Maximum size of an individual log file in kilobytes */ +int log_rotation_size = 0; +/* Maximum lifetime of an individual log file in minutes */ +int log_rotation_age = 0; + +/* Implementation for logging.h */ + +typedef enum +{ + PG_DEBUG, + PG_PROGRESS, + PG_WARNING, + PG_FATAL +} eLogType; + +void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); + +static void elog_internal(int elevel, bool file_only, const char *fmt, va_list args) + pg_attribute_printf(3, 0); +static void elog_stderr(int elevel, const char *fmt, ...) + pg_attribute_printf(2, 3); + +/* Functions to work with log files */ +static void open_logfile(FILE **file, const char *filename_format); +static void release_logfile(void); +static char *logfile_getname(const char *format, time_t timestamp); +static FILE *logfile_open(const char *filename, const char *mode); + +/* Static variables */ + +static FILE *log_file = NULL; +static FILE *error_log_file = NULL; + +static bool exit_hook_registered = false; +/* Logging of the current thread is in progress */ +static bool loggin_in_progress = false; + +static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; + +void +init_logger(const char *root_path) +{ + /* Set log path */ + if (log_level_file != LOG_OFF || error_log_filename) + { + if (log_directory) + strcpy(log_path, log_directory); + else + join_path_components(log_path, root_path, LOG_DIRECTORY_DEFAULT); + } +} + +static void +write_elevel(FILE *stream, int elevel) +{ + switch (elevel) + { + case VERBOSE: + fputs("VERBOSE: ", stream); + break; + case LOG: + fputs("LOG: ", stream); + break; + case INFO: + fputs("INFO: ", stream); + break; + case NOTICE: + fputs("NOTICE: ", stream); + break; + case WARNING: + fputs("WARNING: ", stream); + break; + case ERROR: + fputs("ERROR: ", stream); + break; + default: + elog_stderr(ERROR, "invalid logging level: %d", elevel); + break; + } +} + +/* + * Exit with code if it is an error. + * Check for in_cleanup flag to avoid deadlock in case of ERROR in cleanup + * routines. + */ +static void +exit_if_necessary(int elevel) +{ + if (elevel > WARNING && !in_cleanup) + { + /* Interrupt other possible routines */ + interrupted = true; + + if (loggin_in_progress) + { + loggin_in_progress = false; + pthread_mutex_unlock(&log_file_mutex); + } + + /* If this is not the main thread then don't call exit() */ + if (main_tid != pthread_self()) +#ifdef WIN32 + ExitThread(elevel); +#else + pthread_exit(NULL); +#endif + else + exit(elevel); + } +} + +/* + * Logs to stderr or to log file and exit if ERROR. + * + * Actual implementation for elog() and pg_log(). + */ +static void +elog_internal(int elevel, bool file_only, const char *fmt, va_list args) +{ + bool write_to_file, + write_to_error_log, + write_to_stderr; + va_list error_args, + std_args; + time_t log_time = (time_t) time(NULL); + char strfbuf[128]; + + write_to_file = elevel >= log_level_file && log_path[0] != '\0'; + write_to_error_log = elevel >= ERROR && error_log_filename && + log_path[0] != '\0'; + write_to_stderr = elevel >= log_level_console && !file_only; + + pthread_lock(&log_file_mutex); +#ifdef WIN32 + std_args = NULL; + error_args = NULL; +#endif + loggin_in_progress = true; + + /* We need copy args only if we need write to error log file */ + if (write_to_error_log) + va_copy(error_args, args); + /* + * We need copy args only if we need write to stderr. But do not copy args + * if we need to log only to stderr. + */ + if (write_to_stderr && write_to_file) + va_copy(std_args, args); + + if (write_to_file || write_to_error_log) + strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", + localtime(&log_time)); + + /* + * Write message to log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_file) + { + if (log_file == NULL) + { + if (log_filename == NULL) + open_logfile(&log_file, LOG_FILENAME_DEFAULT); + else + open_logfile(&log_file, log_filename); + } + + fprintf(log_file, "%s: ", strfbuf); + write_elevel(log_file, elevel); + + vfprintf(log_file, fmt, args); + fputc('\n', log_file); + fflush(log_file); + } + + /* + * Write error message to error log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_error_log) + { + if (error_log_file == NULL) + open_logfile(&error_log_file, error_log_filename); + + fprintf(error_log_file, "%s: ", strfbuf); + write_elevel(error_log_file, elevel); + + vfprintf(error_log_file, fmt, error_args); + fputc('\n', error_log_file); + fflush(error_log_file); + + va_end(error_args); + } + + /* + * Write to stderr if the message was not written to log file. + * Write to stderr if the message level is greater than WARNING anyway. + */ + if (write_to_stderr) + { + write_elevel(stderr, elevel); + if (write_to_file) + vfprintf(stderr, fmt, std_args); + else + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + if (write_to_file) + va_end(std_args); + } + + exit_if_necessary(elevel); + + loggin_in_progress = false; + pthread_mutex_unlock(&log_file_mutex); +} + +/* + * Log only to stderr. It is called only within elog_internal() when another + * logging already was started. + */ +static void +elog_stderr(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < ERROR) + return; + + va_start(args, fmt); + + write_elevel(stderr, elevel); + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + va_end(args); + + exit_if_necessary(elevel); +} + +/* + * Logs to stderr or to log file and exit if ERROR. + */ +void +elog(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, false, fmt, args); + va_end(args); +} + +/* + * Logs only to log file and exit if ERROR. + */ +void +elog_file(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, true, fmt, args); + va_end(args); +} + +/* + * Implementation of pg_log() from logging.h. + */ +void +pg_log(eLogType type, const char *fmt, ...) +{ + va_list args; + int elevel = INFO; + + /* Transform logging level from eLogType to utils/logger.h levels */ + switch (type) + { + case PG_DEBUG: + elevel = LOG; + break; + case PG_PROGRESS: + elevel = INFO; + break; + case PG_WARNING: + elevel = WARNING; + break; + case PG_FATAL: + elevel = ERROR; + break; + default: + elog(ERROR, "invalid logging level: %d", type); + break; + } + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, false, fmt, args); + va_end(args); +} + +/* + * Parses string representation of log level. + */ +int +parse_log_level(const char *level) +{ + const char *v = level; + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*v)) + v++; + len = strlen(v); + + if (len == 0) + elog(ERROR, "log-level is empty"); + + if (pg_strncasecmp("off", v, len) == 0) + return LOG_OFF; + else if (pg_strncasecmp("verbose", v, len) == 0) + return VERBOSE; + else if (pg_strncasecmp("log", v, len) == 0) + return LOG; + else if (pg_strncasecmp("info", v, len) == 0) + return INFO; + else if (pg_strncasecmp("notice", v, len) == 0) + return NOTICE; + else if (pg_strncasecmp("warning", v, len) == 0) + return WARNING; + else if (pg_strncasecmp("error", v, len) == 0) + return ERROR; + + /* Log level is invalid */ + elog(ERROR, "invalid log-level \"%s\"", level); + return 0; +} + +/* + * Converts integer representation of log level to string. + */ +const char * +deparse_log_level(int level) +{ + switch (level) + { + case LOG_OFF: + return "OFF"; + case VERBOSE: + return "VERBOSE"; + case LOG: + return "LOG"; + case INFO: + return "INFO"; + case NOTICE: + return "NOTICE"; + case WARNING: + return "WARNING"; + case ERROR: + return "ERROR"; + default: + elog(ERROR, "invalid log-level %d", level); + } + + return NULL; +} + +/* + * Construct logfile name using timestamp information. + * + * Result is palloc'd. + */ +static char * +logfile_getname(const char *format, time_t timestamp) +{ + char *filename; + size_t len; + struct tm *tm = localtime(×tamp); + + if (log_path[0] == '\0') + elog_stderr(ERROR, "logging path is not set"); + + filename = (char *) palloc(MAXPGPATH); + + snprintf(filename, MAXPGPATH, "%s/", log_path); + + len = strlen(filename); + + /* Treat log_filename as a strftime pattern */ + if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) + elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); + + return filename; +} + +/* + * Open a new log file. + */ +static FILE * +logfile_open(const char *filename, const char *mode) +{ + FILE *fh; + + /* + * Create log directory if not present; ignore errors + */ + mkdir(log_path, S_IRWXU); + + fh = fopen(filename, mode); + + if (fh) + setvbuf(fh, NULL, PG_IOLBF, 0); + else + { + int save_errno = errno; + + elog_stderr(ERROR, "could not open log file \"%s\": %s", + filename, strerror(errno)); + errno = save_errno; + } + + return fh; +} + +/* + * Open the log file. + */ +static void +open_logfile(FILE **file, const char *filename_format) +{ + char *filename; + char control[MAXPGPATH]; + struct stat st; + FILE *control_file; + time_t cur_time = time(NULL); + bool rotation_requested = false, + logfile_exists = false; + + filename = logfile_getname(filename_format, cur_time); + + /* "log_path" was checked in logfile_getname() */ + snprintf(control, MAXPGPATH, "%s.rotation", filename); + + if (stat(filename, &st) == -1) + { + if (errno == ENOENT) + { + /* There is no file "filename" and rotation does not need */ + goto logfile_open; + } + else + elog_stderr(ERROR, "cannot stat log file \"%s\": %s", + filename, strerror(errno)); + } + /* Found log file "filename" */ + logfile_exists = true; + + /* First check for rotation */ + if (log_rotation_size > 0 || log_rotation_age > 0) + { + /* Check for rotation by age */ + if (log_rotation_age > 0) + { + struct stat control_st; + + if (stat(control, &control_st) == -1) + { + if (errno != ENOENT) + elog_stderr(ERROR, "cannot stat rotation file \"%s\": %s", + control, strerror(errno)); + } + else + { + char buf[1024]; + + control_file = fopen(control, "r"); + if (control_file == NULL) + elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + if (fgets(buf, lengthof(buf), control_file)) + { + time_t creation_time; + + if (!parse_int64(buf, (int64 *) &creation_time, 0)) + elog_stderr(ERROR, "rotation file \"%s\" has wrong " + "creation timestamp \"%s\"", + control, buf); + /* Parsed creation time */ + + rotation_requested = (cur_time - creation_time) > + /* convert to seconds */ + log_rotation_age * 60; + } + else + elog_stderr(ERROR, "cannot read creation timestamp from " + "rotation file \"%s\"", control); + + fclose(control_file); + } + } + + /* Check for rotation by size */ + if (!rotation_requested && log_rotation_size > 0) + rotation_requested = st.st_size >= + /* convert to bytes */ + log_rotation_size * 1024L; + } + +logfile_open: + if (rotation_requested) + *file = logfile_open(filename, "w"); + else + *file = logfile_open(filename, "a"); + pfree(filename); + + /* Rewrite rotation control file */ + if (rotation_requested || !logfile_exists) + { + time_t timestamp = time(NULL); + + control_file = fopen(control, "w"); + if (control_file == NULL) + elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + fprintf(control_file, "%ld", timestamp); + + fclose(control_file); + } + + /* + * Arrange to close opened file at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(release_logfile); + exit_hook_registered = true; + } +} + +/* + * Closes opened file. + */ +static void +release_logfile(void) +{ + if (log_file) + { + fclose(log_file); + log_file = NULL; + } + if (error_log_file) + { + fclose(error_log_file); + error_log_file = NULL; + } +} diff --git a/src/utils/logger.h b/src/utils/logger.h new file mode 100644 index 00000000..8643ad18 --- /dev/null +++ b/src/utils/logger.h @@ -0,0 +1,54 @@ +/*------------------------------------------------------------------------- + * + * logger.h: - prototypes of logger functions. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef LOGGER_H +#define LOGGER_H + +#include "postgres_fe.h" + +#define LOG_NONE (-10) + +/* Log level */ +#define VERBOSE (-5) +#define LOG (-4) +#define INFO (-3) +#define NOTICE (-2) +#define WARNING (-1) +#define ERROR 1 +#define LOG_OFF 10 + +/* Logger parameters */ + +extern int log_to_file; +extern int log_level_console; +extern int log_level_file; + +extern char *log_filename; +extern char *error_log_filename; +extern char *log_directory; +extern char log_path[MAXPGPATH]; + +#define LOG_ROTATION_SIZE_DEFAULT 0 +#define LOG_ROTATION_AGE_DEFAULT 0 +extern int log_rotation_size; +extern int log_rotation_age; + +#define LOG_LEVEL_CONSOLE_DEFAULT INFO +#define LOG_LEVEL_FILE_DEFAULT LOG_OFF + +#undef elog +extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); +extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); + +extern void init_logger(const char *root_path); + +extern int parse_log_level(const char *level); +extern const char *deparse_log_level(int level); + +#endif /* LOGGER_H */ diff --git a/src/utils/parray.c b/src/utils/parray.c new file mode 100644 index 00000000..a9ba7c8e --- /dev/null +++ b/src/utils/parray.c @@ -0,0 +1,196 @@ +/*------------------------------------------------------------------------- + * + * parray.c: pointer array collection. + * + * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "src/pg_probackup.h" + +/* members of struct parray are hidden from client. */ +struct parray +{ + void **data; /* poiter array, expanded if necessary */ + size_t alloced; /* number of elements allocated */ + size_t used; /* number of elements in use */ +}; + +/* + * Create new parray object. + * Never returns NULL. + */ +parray * +parray_new(void) +{ + parray *a = pgut_new(parray); + + a->data = NULL; + a->used = 0; + a->alloced = 0; + + parray_expand(a, 1024); + + return a; +} + +/* + * Expand array pointed by data to newsize. + * Elements in expanded area are initialized to NULL. + * Note: never returns NULL. + */ +void +parray_expand(parray *array, size_t newsize) +{ + void **p; + + /* already allocated */ + if (newsize <= array->alloced) + return; + + p = pgut_realloc(array->data, sizeof(void *) * newsize); + + /* initialize expanded area to NULL */ + memset(p + array->alloced, 0, (newsize - array->alloced) * sizeof(void *)); + + array->alloced = newsize; + array->data = p; +} + +void +parray_free(parray *array) +{ + if (array == NULL) + return; + free(array->data); + free(array); +} + +void +parray_append(parray *array, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + array->data[array->used++] = elem; +} + +void +parray_insert(parray *array, size_t index, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + memmove(array->data + index + 1, array->data + index, + (array->alloced - index - 1) * sizeof(void *)); + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; + else + array->used++; +} + +/* + * Concatinate two parray. + * parray_concat() appends the copy of the content of src to the end of dest. + */ +parray * +parray_concat(parray *dest, const parray *src) +{ + /* expand head array */ + parray_expand(dest, dest->used + src->used); + + /* copy content of src after content of dest */ + memcpy(dest->data + dest->used, src->data, src->used * sizeof(void *)); + dest->used += parray_num(src); + + return dest; +} + +void +parray_set(parray *array, size_t index, void *elem) +{ + if (index > array->alloced - 1) + parray_expand(array, index + 1); + + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; +} + +void * +parray_get(const parray *array, size_t index) +{ + if (index > array->alloced - 1) + return NULL; + return array->data[index]; +} + +void * +parray_remove(parray *array, size_t index) +{ + void *val; + + /* removing unused element */ + if (index > array->used) + return NULL; + + val = array->data[index]; + + /* Do not move if the last element was removed. */ + if (index < array->alloced - 1) + memmove(array->data + index, array->data + index + 1, + (array->alloced - index - 1) * sizeof(void *)); + + /* adjust used count */ + array->used--; + + return val; +} + +bool +parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + int i; + + for (i = 0; i < array->used; i++) + { + if (compare(&key, &array->data[i]) == 0) + { + parray_remove(array, i); + return true; + } + } + return false; +} + +size_t +parray_num(const parray *array) +{ + return array->used; +} + +void +parray_qsort(parray *array, int(*compare)(const void *, const void *)) +{ + qsort(array->data, array->used, sizeof(void *), compare); +} + +void +parray_walk(parray *array, void (*action)(void *)) +{ + int i; + for (i = 0; i < array->used; i++) + action(array->data[i]); +} + +void * +parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + return bsearch(&key, array->data, array->used, sizeof(void *), compare); +} diff --git a/src/utils/parray.h b/src/utils/parray.h new file mode 100644 index 00000000..833a6961 --- /dev/null +++ b/src/utils/parray.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * parray.h: pointer array collection. + * + * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#ifndef PARRAY_H +#define PARRAY_H + +/* + * "parray" hold pointers to objects in a linear memory area. + * Client use "parray *" to access parray object. + */ +typedef struct parray parray; + +extern parray *parray_new(void); +extern void parray_expand(parray *array, size_t newnum); +extern void parray_free(parray *array); +extern void parray_append(parray *array, void *val); +extern void parray_insert(parray *array, size_t index, void *val); +extern parray *parray_concat(parray *head, const parray *tail); +extern void parray_set(parray *array, size_t index, void *val); +extern void *parray_get(const parray *array, size_t index); +extern void *parray_remove(parray *array, size_t index); +extern bool parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern size_t parray_num(const parray *array); +extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); +extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern void parray_walk(parray *array, void (*action)(void *)); + +#endif /* PARRAY_H */ + diff --git a/src/utils/pgut.c b/src/utils/pgut.c new file mode 100644 index 00000000..f341c6a4 --- /dev/null +++ b/src/utils/pgut.c @@ -0,0 +1,2417 @@ +/*------------------------------------------------------------------------- + * + * pgut.c + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" +#include "libpq/pqsignal.h" + +#include "getopt_long.h" +#include +#include +#include + +#include "logger.h" +#include "pgut.h" + +/* old gcc doesn't have LLONG_MAX. */ +#ifndef LLONG_MAX +#if defined(HAVE_LONG_INT_64) || !defined(HAVE_LONG_LONG_INT_64) +#define LLONG_MAX LONG_MAX +#else +#define LLONG_MAX INT64CONST(0x7FFFFFFFFFFFFFFF) +#endif +#endif + +#define MAX_TZDISP_HOUR 15 /* maximum allowed hour part */ +#define SECS_PER_MINUTE 60 +#define MINS_PER_HOUR 60 +#define MAXPG_LSNCOMPONENT 8 + +const char *PROGRAM_NAME = NULL; + +const char *pgut_dbname = NULL; +const char *host = NULL; +const char *port = NULL; +const char *username = NULL; +static char *password = NULL; +bool prompt_password = true; +bool force_password = false; + +/* Database connections */ +static PGcancel *volatile cancel_conn = NULL; + +/* Interrupted by SIGINT (Ctrl+C) ? */ +bool interrupted = false; +bool in_cleanup = false; +bool in_password = false; + +static bool parse_pair(const char buffer[], char key[], char value[]); + +/* Connection routines */ +static void init_cancel_handler(void); +static void on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn); +static void on_after_exec(PGcancel *thread_cancel_conn); +static void on_interrupt(void); +static void on_cleanup(void); +static void exit_or_abort(int exitcode); +static const char *get_username(void); +static pqsigfunc oldhandler = NULL; + +/* + * Unit conversion tables. + * + * Copied from guc.c. + */ +#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ + +typedef struct +{ + char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or + * "min" */ + int base_unit; /* OPTION_UNIT_XXX */ + int multiplier; /* If positive, multiply the value with this + * for unit -> base_unit conversion. If + * negative, divide (with the absolute value) */ +} unit_conversion; + +static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; + +static const unit_conversion memory_unit_conversion_table[] = +{ + {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, + {"GB", OPTION_UNIT_KB, 1024 * 1024}, + {"MB", OPTION_UNIT_KB, 1024}, + {"KB", OPTION_UNIT_KB, 1}, + {"kB", OPTION_UNIT_KB, 1}, + + {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, + + {""} /* end of table marker */ +}; + +static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; + +static const unit_conversion time_unit_conversion_table[] = +{ + {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, + {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, + {"min", OPTION_UNIT_MS, 1000 * 60}, + {"s", OPTION_UNIT_MS, 1000}, + {"ms", OPTION_UNIT_MS, 1}, + + {"d", OPTION_UNIT_S, 60 * 60 * 24}, + {"h", OPTION_UNIT_S, 60 * 60}, + {"min", OPTION_UNIT_S, 60}, + {"s", OPTION_UNIT_S, 1}, + {"ms", OPTION_UNIT_S, -1000}, + + {"d", OPTION_UNIT_MIN, 60 * 24}, + {"h", OPTION_UNIT_MIN, 60}, + {"min", OPTION_UNIT_MIN, 1}, + {"s", OPTION_UNIT_MIN, -60}, + {"ms", OPTION_UNIT_MIN, -1000 * 60}, + + {""} /* end of table marker */ +}; + +static size_t +option_length(const pgut_option opts[]) +{ + size_t len; + + for (len = 0; opts && opts[len].type; len++) { } + + return len; +} + +static int +option_has_arg(char type) +{ + switch (type) + { + case 'b': + case 'B': + return no_argument; + default: + return required_argument; + } +} + +static void +option_copy(struct option dst[], const pgut_option opts[], size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) + { + dst[i].name = opts[i].lname; + dst[i].has_arg = option_has_arg(opts[i].type); + dst[i].flag = NULL; + dst[i].val = opts[i].sname; + } +} + +static pgut_option * +option_find(int c, pgut_option opts1[]) +{ + size_t i; + + for (i = 0; opts1 && opts1[i].type; i++) + if (opts1[i].sname == c) + return &opts1[i]; + + return NULL; /* not found */ +} + +static void +assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) +{ + const char *message; + + if (opt == NULL) + { + fprintf(stderr, "Try \"%s --help\" for more information.\n", PROGRAM_NAME); + exit_or_abort(ERROR); + } + + if (opt->source > src) + { + /* high prior value has been set already. */ + return; + } + /* Allow duplicate entries for function option */ + else if (src >= SOURCE_CMDLINE && opt->source >= src && opt->type != 'f') + { + message = "specified only once"; + } + else + { + pgut_optsrc orig_source = opt->source; + + /* can be overwritten if non-command line source */ + opt->source = src; + + switch (opt->type) + { + case 'b': + case 'B': + if (optarg == NULL) + { + *((bool *) opt->var) = (opt->type == 'b'); + return; + } + else if (parse_bool(optarg, (bool *) opt->var)) + { + return; + } + message = "a boolean"; + break; + case 'f': + ((pgut_optfn) opt->var)(opt, optarg); + return; + case 'i': + if (parse_int32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit signed integer"; + break; + case 'u': + if (parse_uint32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit unsigned integer"; + break; + case 'I': + if (parse_int64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit signed integer"; + break; + case 'U': + if (parse_uint64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit unsigned integer"; + break; + case 's': + if (orig_source != SOURCE_DEFAULT) + free(*(char **) opt->var); + *(char **) opt->var = pgut_strdup(optarg); + if (strcmp(optarg,"") != 0) + return; + message = "a valid string. But provided: "; + break; + case 't': + if (parse_time(optarg, opt->var, + opt->source == SOURCE_FILE)) + return; + message = "a time"; + break; + default: + elog(ERROR, "invalid option type: %c", opt->type); + return; /* keep compiler quiet */ + } + } + + if (isprint(opt->sname)) + elog(ERROR, "option -%c, --%s should be %s: '%s'", + opt->sname, opt->lname, message, optarg); + else + elog(ERROR, "option --%s should be %s: '%s'", + opt->lname, message, optarg); +} + +/* + * Convert a value from one of the human-friendly units ("kB", "min" etc.) + * to the given base unit. 'value' and 'unit' are the input value and unit + * to convert from. The converted value is stored in *base_value. + * + * Returns true on success, false if the input unit is not recognized. + */ +static bool +convert_to_base_unit(int64 value, const char *unit, + int base_unit, int64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Unsigned variant of convert_to_base_unit() + */ +static bool +convert_to_base_unit_u(uint64 value, const char *unit, + int base_unit, uint64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Convert a value in some base unit to a human-friendly unit. The output + * unit is chosen so that it's the greatest unit that can represent the value + * without loss. For example, if the base unit is GUC_UNIT_KB, 1024 is + * converted to 1 MB, but 1025 is represented as 1025 kB. + */ +void +convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +/* + * Unsigned variant of convert_from_base_unit() + */ +void +convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +static bool +parse_unit(char *unit_str, int flags, int64 value, int64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Unsigned variant of parse_unit() + */ +static bool +parse_unit_u(char *unit_str, int flags, uint64 value, uint64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit_u(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Try to interpret value as boolean value. Valid values are: true, + * false, yes, no, on, off, 1, 0; as well as unique prefixes thereof. + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + */ +bool +parse_bool(const char *value, bool *result) +{ + return parse_bool_with_len(value, strlen(value), result); +} + +bool +parse_bool_with_len(const char *value, size_t len, bool *result) +{ + switch (*value) + { + case 't': + case 'T': + if (pg_strncasecmp(value, "true", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'f': + case 'F': + if (pg_strncasecmp(value, "false", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'y': + case 'Y': + if (pg_strncasecmp(value, "yes", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'n': + case 'N': + if (pg_strncasecmp(value, "no", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'o': + case 'O': + /* 'o' is not unique enough */ + if (pg_strncasecmp(value, "on", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = true; + return true; + } + else if (pg_strncasecmp(value, "off", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = false; + return true; + } + break; + case '1': + if (len == 1) + { + if (result) + *result = true; + return true; + } + break; + case '0': + if (len == 1) + { + if (result) + *result = false; + return true; + } + break; + default: + break; + } + + if (result) + *result = false; /* suppress compiler warning */ + return false; +} + +/* + * Parse string as 32bit signed int. + * valid range: -2147483648 ~ 2147483647 + */ +bool +parse_int32(const char *value, int32 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = INT_MAX; + return true; + } + + errno = 0; + val = strtol(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE || val != (int64) ((int32) val)) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as 32bit unsigned int. + * valid range: 0 ~ 4294967295 (2^32-1) + */ +bool +parse_uint32(const char *value, uint32 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = UINT_MAX; + return true; + } + + errno = 0; + val = strtoul(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE || val != (uint64) ((uint32) val)) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as int64 + * valid range: -9223372036854775808 ~ 9223372036854775807 + */ +bool +parse_int64(const char *value, int64 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = LLONG_MAX; + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtol(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoll(value, &endptr, 0); +#else + val = strtol(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as uint64 + * valid range: 0 ~ (2^64-1) + */ +bool +parse_uint64(const char *value, uint64 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { +#if defined(HAVE_LONG_INT_64) + *result = ULONG_MAX; +#elif defined(HAVE_LONG_LONG_INT_64) + *result = ULLONG_MAX; +#else + *result = ULONG_MAX; +#endif + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtoul(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoull(value, &endptr, 0); +#else + val = strtoul(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Convert ISO-8601 format string to time_t value. + * + * If utc_default is true, then if timezone offset isn't specified tz will be + * +00:00. + */ +bool +parse_time(const char *value, time_t *result, bool utc_default) +{ + size_t len; + int fields_num, + tz = 0, + i; + bool tz_set = false; + char *tmp; + struct tm tm; + char junk[2]; + + /* tmp = replace( value, !isalnum, ' ' ) */ + tmp = pgut_malloc(strlen(value) + + 1); + len = 0; + fields_num = 1; + + while (*value) + { + if (IsAlnum(*value)) + { + tmp[len++] = *value; + value++; + } + else if (fields_num < 6) + { + fields_num++; + tmp[len++] = ' '; + value++; + } + /* timezone field is 7th */ + else if ((*value == '-' || *value == '+') && fields_num == 6) + { + int hr, + min, + sec = 0; + char *cp; + + errno = 0; + hr = strtol(value + 1, &cp, 10); + if ((value + 1) == cp || errno == ERANGE) + return false; + + /* explicit delimiter? */ + if (*cp == ':') + { + errno = 0; + min = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + if (*cp == ':') + { + errno = 0; + sec = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + } + } + /* otherwise, might have run things together... */ + else if (*cp == '\0' && strlen(value) > 3) + { + min = hr % 100; + hr = hr / 100; + /* we could, but don't, support a run-together hhmmss format */ + } + else + min = 0; + + /* Range-check the values; see notes in datatype/timestamp.h */ + if (hr < 0 || hr > MAX_TZDISP_HOUR) + return false; + if (min < 0 || min >= MINS_PER_HOUR) + return false; + if (sec < 0 || sec >= SECS_PER_MINUTE) + return false; + + tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; + if (*value == '-') + tz = -tz; + + tz_set = true; + + fields_num++; + value = cp; + } + /* wrong format */ + else if (!IsSpace(*value)) + return false; + } + tmp[len] = '\0'; + + /* parse for "YYYY-MM-DD HH:MI:SS" */ + memset(&tm, 0, sizeof(tm)); + tm.tm_year = 0; /* tm_year is year - 1900 */ + tm.tm_mon = 0; /* tm_mon is 0 - 11 */ + tm.tm_mday = 1; /* tm_mday is 1 - 31 */ + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); + free(tmp); + + if (i < 1 || 6 < i) + return false; + + /* adjust year */ + if (tm.tm_year < 100) + tm.tm_year += 2000 - 1900; + else if (tm.tm_year >= 1900) + tm.tm_year -= 1900; + + /* adjust month */ + if (i > 1) + tm.tm_mon -= 1; + + /* determine whether Daylight Saving Time is in effect */ + tm.tm_isdst = -1; + + *result = mktime(&tm); + + /* adjust time zone */ + if (tz_set || utc_default) + { + time_t ltime = time(NULL); + struct tm *ptm = gmtime(<ime); + time_t gmt = mktime(ptm); + time_t offset; + + /* UTC time */ + *result -= tz; + + /* Get local time */ + ptm = localtime(<ime); + offset = ltime - gmt + (ptm->tm_isdst ? 3600 : 0); + + *result += offset; + } + + return true; +} + +/* + * Try to parse value as an integer. The accepted formats are the + * usual decimal, octal, or hexadecimal formats, optionally followed by + * a unit name if "flags" indicates a unit is allowed. + * + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable + * HINT message, or NULL if no hint provided. + */ +bool +parse_int(const char *value, int *result, int flags, const char **hintmsg) +{ + int64 val; + char *endptr; + + /* To suppress compiler warnings, always set output params */ + if (result) + *result = 0; + if (hintmsg) + *hintmsg = NULL; + + /* We assume here that int64 is at least as wide as long */ + errno = 0; + val = strtol(value, &endptr, 0); + + if (endptr == value) + return false; /* no HINT for integer syntax error */ + + if (errno == ERANGE || val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + /* Handle possible unit */ + if (*endptr != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*endptr != '\0' && !isspace((unsigned char) *endptr) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(endptr++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + if (*endptr == '\0') + converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), + &val); + if (!converted) + { + /* invalid unit, or garbage after the unit; set hint and fail. */ + if (hintmsg) + { + if (flags & OPTION_UNIT_MEMORY) + *hintmsg = memory_units_hint; + else + *hintmsg = time_units_hint; + } + return false; + } + + /* Check for overflow due to units conversion */ + if (val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + } + + if (result) + *result = (int) val; + return true; +} + +bool +parse_lsn(const char *value, XLogRecPtr *result) +{ + uint32 xlogid; + uint32 xrecoff; + int len1; + int len2; + + len1 = strspn(value, "0123456789abcdefABCDEF"); + if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') + elog(ERROR, "invalid LSN \"%s\"", value); + len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); + if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') + elog(ERROR, "invalid LSN \"%s\"", value); + + if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) + *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + { + elog(ERROR, "invalid LSN \"%s\"", value); + return false; + } + + return true; +} + +static char * +longopts_to_optstring(const struct option opts[], const size_t len) +{ + size_t i; + char *result; + char *s; + + result = pgut_malloc(len * 2 + 1); + + s = result; + for (i = 0; i < len; i++) + { + if (!isprint(opts[i].val)) + continue; + *s++ = opts[i].val; + if (opts[i].has_arg != no_argument) + *s++ = ':'; + } + *s = '\0'; + + return result; +} + +void +pgut_getopt_env(pgut_option options[]) +{ + size_t i; + + for (i = 0; options && options[i].type; i++) + { + pgut_option *opt = &options[i]; + const char *value = NULL; + + /* If option was already set do not check env */ + if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV) + continue; + + if (strcmp(opt->lname, "pgdata") == 0) + value = getenv("PGDATA"); + if (strcmp(opt->lname, "port") == 0) + value = getenv("PGPORT"); + if (strcmp(opt->lname, "host") == 0) + value = getenv("PGHOST"); + if (strcmp(opt->lname, "username") == 0) + value = getenv("PGUSER"); + if (strcmp(opt->lname, "pgdatabase") == 0) + { + value = getenv("PGDATABASE"); + if (value == NULL) + value = getenv("PGUSER"); + if (value == NULL) + value = get_username(); + } + + if (value) + assign_option(opt, value, SOURCE_ENV); + } +} + +int +pgut_getopt(int argc, char **argv, pgut_option options[]) +{ + int c; + int optindex = 0; + char *optstring; + pgut_option *opt; + struct option *longopts; + size_t len; + + len = option_length(options); + longopts = pgut_newarray(struct option, len + 1 /* zero/end option */); + option_copy(longopts, options, len); + + optstring = longopts_to_optstring(longopts, len); + + /* Assign named options */ + while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) + { + opt = option_find(c, options); + if (opt && opt->allowed < SOURCE_CMDLINE) + elog(ERROR, "option %s cannot be specified in command line", + opt->lname); + /* Check 'opt == NULL' is performed in assign_option() */ + assign_option(opt, optarg, SOURCE_CMDLINE); + } + + init_cancel_handler(); + atexit(on_cleanup); + + return optind; +} + +/* compare two strings ignore cases and ignore -_ */ +static bool +key_equals(const char *lhs, const char *rhs) +{ + for (; *lhs && *rhs; lhs++, rhs++) + { + if (strchr("-_ ", *lhs)) + { + if (!strchr("-_ ", *rhs)) + return false; + } + else if (ToLower(*lhs) != ToLower(*rhs)) + return false; + } + + return *lhs == '\0' && *rhs == '\0'; +} + +/* + * Get configuration from configuration file. + * Return number of parsed options + */ +int +pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict) +{ + FILE *fp; + char buf[1024]; + char key[1024]; + char value[1024]; + int parsed_options = 0; + + if (!options) + return parsed_options; + + if ((fp = pgut_fopen(path, "rt", true)) == NULL) + return parsed_options; + + while (fgets(buf, lengthof(buf), fp)) + { + size_t i; + + for (i = strlen(buf); i > 0 && IsSpace(buf[i - 1]); i--) + buf[i - 1] = '\0'; + + if (parse_pair(buf, key, value)) + { + for (i = 0; options[i].type; i++) + { + pgut_option *opt = &options[i]; + + if (key_equals(key, opt->lname)) + { + if (opt->allowed < SOURCE_FILE && + opt->allowed != SOURCE_FILE_STRICT) + elog(elevel, "option %s cannot be specified in file", opt->lname); + else if (opt->source <= SOURCE_FILE) + { + assign_option(opt, value, SOURCE_FILE); + parsed_options++; + } + break; + } + } + if (strict && !options[i].type) + elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path); + } + } + + fclose(fp); + + return parsed_options; +} + +static const char * +skip_space(const char *str, const char *line) +{ + while (IsSpace(*str)) { str++; } + return str; +} + +static const char * +get_next_token(const char *src, char *dst, const char *line) +{ + const char *s; + int i; + int j; + + if ((s = skip_space(src, line)) == NULL) + return NULL; + + /* parse quoted string */ + if (*s == '\'') + { + s++; + for (i = 0, j = 0; s[i] != '\0'; i++) + { + if (s[i] == '\\') + { + i++; + switch (s[i]) + { + case 'b': + dst[j] = '\b'; + break; + case 'f': + dst[j] = '\f'; + break; + case 'n': + dst[j] = '\n'; + break; + case 'r': + dst[j] = '\r'; + break; + case 't': + dst[j] = '\t'; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { + int k; + long octVal = 0; + + for (k = 0; + s[i + k] >= '0' && s[i + k] <= '7' && k < 3; + k++) + octVal = (octVal << 3) + (s[i + k] - '0'); + i += k - 1; + dst[j] = ((char) octVal); + } + break; + default: + dst[j] = s[i]; + break; + } + } + else if (s[i] == '\'') + { + i++; + /* doubled quote becomes just one quote */ + if (s[i] == '\'') + dst[j] = s[i]; + else + break; + } + else + dst[j] = s[i]; + j++; + } + } + else + { + i = j = strcspn(s, "#\n\r\t\v"); + memcpy(dst, s, j); + } + + dst[j] = '\0'; + return s + i; +} + +static bool +parse_pair(const char buffer[], char key[], char value[]) +{ + const char *start; + const char *end; + + key[0] = value[0] = '\0'; + + /* + * parse key + */ + start = buffer; + if ((start = skip_space(start, buffer)) == NULL) + return false; + + end = start + strcspn(start, "=# \n\r\t\v"); + + /* skip blank buffer */ + if (end - start <= 0) + { + if (*start == '=') + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + /* key found */ + strncpy(key, start, end - start); + key[end - start] = '\0'; + + /* find key and value split char */ + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '=') + { + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + start++; + + /* + * parse value + */ + if ((end = get_next_token(start, value, buffer)) == NULL) + return false; + + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '\0' && *start != '#') + { + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + return true; +} + +/* + * Ask the user for a password; 'username' is the username the + * password is for, if one has been explicitly specified. + * Set malloc'd string to the global variable 'password'. + */ +static void +prompt_for_password(const char *username) +{ + in_password = true; + + if (password) + { + free(password); + password = NULL; + } + +#if PG_VERSION_NUM >= 100000 + password = (char *) pgut_malloc(sizeof(char) * 100 + 1); + if (username == NULL) + simple_prompt("Password: ", password, 100, false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + simple_prompt(message, password, 100, false); + } +#else + if (username == NULL) + password = simple_prompt("Password: ", 100, false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + password = simple_prompt(message, 100, false); + } +#endif + + in_password = false; +} + +/* + * Copied from pg_basebackup.c + * Escape a parameter value so that it can be used as part of a libpq + * connection string, e.g. in: + * + * application_name= + * + * The returned string is malloc'd. Return NULL on out-of-memory. + */ +static char * +escapeConnectionParameter(const char *src) +{ + bool need_quotes = false; + bool need_escaping = false; + const char *p; + char *dstbuf; + char *dst; + + /* + * First check if quoting is needed. Any quote (') or backslash (\) + * characters need to be escaped. Parameters are separated by whitespace, + * so any string containing whitespace characters need to be quoted. An + * empty string is represented by ''. + */ + if (strchr(src, '\'') != NULL || strchr(src, '\\') != NULL) + need_escaping = true; + + for (p = src; *p; p++) + { + if (isspace((unsigned char) *p)) + { + need_quotes = true; + break; + } + } + + if (*src == '\0') + return pg_strdup("''"); + + if (!need_quotes && !need_escaping) + return pg_strdup(src); /* no quoting or escaping needed */ + + /* + * Allocate a buffer large enough for the worst case that all the source + * characters need to be escaped, plus quotes. + */ + dstbuf = pg_malloc(strlen(src) * 2 + 2 + 1); + + dst = dstbuf; + if (need_quotes) + *(dst++) = '\''; + for (; *src; src++) + { + if (*src == '\'' || *src == '\\') + *(dst++) = '\\'; + *(dst++) = *src; + } + if (need_quotes) + *(dst++) = '\''; + *dst = '\0'; + + return dstbuf; +} + +/* Construct a connection string for possible future use in recovery.conf */ +char * +pgut_get_conninfo_string(PGconn *conn) +{ + PQconninfoOption *connOptions; + PQconninfoOption *option; + PQExpBuffer buf = createPQExpBuffer(); + char *connstr; + bool firstkeyword = true; + char *escaped; + + connOptions = PQconninfo(conn); + if (connOptions == NULL) + elog(ERROR, "out of memory"); + + /* Construct a new connection string in key='value' format. */ + for (option = connOptions; option && option->keyword; option++) + { + /* + * Do not emit this setting if: - the setting is "replication", + * "dbname" or "fallback_application_name", since these would be + * overridden by the libpqwalreceiver module anyway. - not set or + * empty. + */ + if (strcmp(option->keyword, "replication") == 0 || + strcmp(option->keyword, "dbname") == 0 || + strcmp(option->keyword, "fallback_application_name") == 0 || + (option->val == NULL) || + (option->val != NULL && option->val[0] == '\0')) + continue; + + /* do not print password into the file */ + if (strcmp(option->keyword, "password") == 0) + continue; + + if (!firstkeyword) + appendPQExpBufferChar(buf, ' '); + + firstkeyword = false; + + escaped = escapeConnectionParameter(option->val); + appendPQExpBuffer(buf, "%s=%s", option->keyword, escaped); + free(escaped); + } + + connstr = pg_strdup(buf->data); + destroyPQExpBuffer(buf); + return connstr; +} + +PGconn * +pgut_connect(const char *dbname) +{ + return pgut_connect_extended(host, port, dbname, username); +} + +PGconn * +pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login) +{ + PGconn *conn; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + if (force_password && !prompt_password) + elog(ERROR, "You cannot specify --password and --no-password options together"); + + if (!password && force_password) + prompt_for_password(login); + + /* Start the connection. Loop until we have a password if requested by backend. */ + for (;;) + { + conn = PQsetdbLogin(pghost, pgport, NULL, NULL, + dbname, login, password); + + if (PQstatus(conn) == CONNECTION_OK) + return conn; + + if (conn && PQconnectionNeedsPassword(conn) && prompt_password) + { + PQfinish(conn); + prompt_for_password(login); + + if (interrupted) + elog(ERROR, "interrupted"); + + if (password == NULL || password[0] == '\0') + elog(ERROR, "no password supplied"); + + continue; + } + elog(ERROR, "could not connect to database %s: %s", + dbname, PQerrorMessage(conn)); + + PQfinish(conn); + return NULL; + } +} + +PGconn * +pgut_connect_replication(const char *dbname) +{ + return pgut_connect_replication_extended(host, port, dbname, username); +} + +PGconn * +pgut_connect_replication_extended(const char *pghost, const char *pgport, + const char *dbname, const char *pguser) +{ + PGconn *tmpconn; + int argcount = 7; /* dbname, replication, fallback_app_name, + * host, user, port, password */ + int i; + const char **keywords; + const char **values; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + if (force_password && !prompt_password) + elog(ERROR, "You cannot specify --password and --no-password options together"); + + if (!password && force_password) + prompt_for_password(pguser); + + i = 0; + + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + + keywords[i] = "dbname"; + values[i] = "replication"; + i++; + keywords[i] = "replication"; + values[i] = "true"; + i++; + keywords[i] = "fallback_application_name"; + values[i] = PROGRAM_NAME; + i++; + + if (pghost) + { + keywords[i] = "host"; + values[i] = pghost; + i++; + } + if (pguser) + { + keywords[i] = "user"; + values[i] = pguser; + i++; + } + if (pgport) + { + keywords[i] = "port"; + values[i] = pgport; + i++; + } + + /* Use (or reuse, on a subsequent connection) password if we have it */ + if (password) + { + keywords[i] = "password"; + values[i] = password; + } + else + { + keywords[i] = NULL; + values[i] = NULL; + } + + for (;;) + { + tmpconn = PQconnectdbParams(keywords, values, true); + + + if (PQstatus(tmpconn) == CONNECTION_OK) + { + free(values); + free(keywords); + return tmpconn; + } + + if (tmpconn && PQconnectionNeedsPassword(tmpconn) && prompt_password) + { + PQfinish(tmpconn); + prompt_for_password(pguser); + keywords[i] = "password"; + values[i] = password; + continue; + } + + elog(ERROR, "could not connect to database %s: %s", + dbname, PQerrorMessage(tmpconn)); + PQfinish(tmpconn); + free(values); + free(keywords); + return NULL; + } +} + + +void +pgut_disconnect(PGconn *conn) +{ + if (conn) + PQfinish(conn); +} + +/* set/get host and port for connecting standby server */ +const char * +pgut_get_host() +{ + return host; +} + +const char * +pgut_get_port() +{ + return port; +} + +void +pgut_set_host(const char *new_host) +{ + host = new_host; +} + +void +pgut_set_port(const char *new_port) +{ + port = new_port; +} + + +PGresult * +pgut_execute_parallel(PGconn* conn, + PGcancel* thread_cancel_conn, const char *query, + int nParams, const char **params, + bool text_result) +{ + PGresult *res; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(ERROR, "not connected"); + return NULL; + } + + //on_before_exec(conn, thread_cancel_conn); + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + //on_after_exec(thread_cancel_conn); + + switch (PQresultStatus(res)) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + case PGRES_COPY_IN: + break; + default: + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + break; + } + + return res; +} + +PGresult * +pgut_execute(PGconn* conn, const char *query, int nParams, const char **params) +{ + return pgut_execute_extended(conn, query, nParams, params, true, false); +} + +PGresult * +pgut_execute_extended(PGconn* conn, const char *query, int nParams, + const char **params, bool text_result, bool ok_error) +{ + PGresult *res; + ExecStatusType res_status; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(ERROR, "not connected"); + return NULL; + } + + on_before_exec(conn, NULL); + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + on_after_exec(NULL); + + res_status = PQresultStatus(res); + switch (res_status) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + case PGRES_COPY_IN: + break; + default: + if (ok_error && res_status == PGRES_FATAL_ERROR) + break; + + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + break; + } + + return res; +} + +bool +pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel) +{ + int res; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(elevel, "not connected"); + return false; + } + + if (nParams == 0) + res = PQsendQuery(conn, query); + else + res = PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, 0); + + if (res != 1) + { + elog(elevel, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + return false; + } + + return true; +} + +void +pgut_cancel(PGconn* conn) +{ + PGcancel *cancel_conn = PQgetCancel(conn); + char errbuf[256]; + + if (cancel_conn != NULL) + { + if (PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + elog(WARNING, "Cancel request sent"); + else + elog(WARNING, "Cancel request failed"); + } + + if (cancel_conn) + PQfreeCancel(cancel_conn); +} + +int +pgut_wait(int num, PGconn *connections[], struct timeval *timeout) +{ + /* all connections are busy. wait for finish */ + while (!interrupted) + { + int i; + fd_set mask; + int maxsock; + + FD_ZERO(&mask); + + maxsock = -1; + for (i = 0; i < num; i++) + { + int sock; + + if (connections[i] == NULL) + continue; + sock = PQsocket(connections[i]); + if (sock >= 0) + { + FD_SET(sock, &mask); + if (maxsock < sock) + maxsock = sock; + } + } + + if (maxsock == -1) + { + errno = ENOENT; + return -1; + } + + i = wait_for_sockets(maxsock + 1, &mask, timeout); + if (i == 0) + break; /* timeout */ + + for (i = 0; i < num; i++) + { + if (connections[i] && FD_ISSET(PQsocket(connections[i]), &mask)) + { + PQconsumeInput(connections[i]); + if (PQisBusy(connections[i])) + continue; + return i; + } + } + } + + errno = EINTR; + return -1; +} + +#ifdef WIN32 +static CRITICAL_SECTION cancelConnLock; +#endif + +/* + * on_before_exec + * + * Set cancel_conn to point to the current database connection. + */ +static void +on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn) +{ + PGcancel *old; + + if (in_cleanup) + return; /* forbid cancel during cleanup */ + +#ifdef WIN32 + EnterCriticalSection(&cancelConnLock); +#endif + + if (thread_cancel_conn) + { + //elog(WARNING, "Handle tread_cancel_conn. on_before_exec"); + old = thread_cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + thread_cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + + thread_cancel_conn = PQgetCancel(conn); + } + else + { + /* Free the old one if we have one */ + old = cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + + cancel_conn = PQgetCancel(conn); + } + +#ifdef WIN32 + LeaveCriticalSection(&cancelConnLock); +#endif +} + +/* + * on_after_exec + * + * Free the current cancel connection, if any, and set to NULL. + */ +static void +on_after_exec(PGcancel *thread_cancel_conn) +{ + PGcancel *old; + + if (in_cleanup) + return; /* forbid cancel during cleanup */ + +#ifdef WIN32 + EnterCriticalSection(&cancelConnLock); +#endif + + if (thread_cancel_conn) + { + //elog(WARNING, "Handle tread_cancel_conn. on_after_exec"); + old = thread_cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + thread_cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + } + else + { + old = cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + } +#ifdef WIN32 + LeaveCriticalSection(&cancelConnLock); +#endif +} + +/* + * Handle interrupt signals by cancelling the current command. + */ +static void +on_interrupt(void) +{ + int save_errno = errno; + char errbuf[256]; + + /* Set interruped flag */ + interrupted = true; + + /* User promts password, call on_cleanup() byhand */ + if (in_password) + { + on_cleanup(); + + pqsignal(SIGINT, oldhandler); + kill(0, SIGINT); + } + + /* Send QueryCancel if we are processing a database query */ + if (!in_cleanup && cancel_conn != NULL && + PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + { + elog(WARNING, "Cancel request sent"); + } + + errno = save_errno; /* just in case the write changed it */ +} + +typedef struct pgut_atexit_item pgut_atexit_item; +struct pgut_atexit_item +{ + pgut_atexit_callback callback; + void *userdata; + pgut_atexit_item *next; +}; + +static pgut_atexit_item *pgut_atexit_stack = NULL; + +void +pgut_atexit_push(pgut_atexit_callback callback, void *userdata) +{ + pgut_atexit_item *item; + + AssertArg(callback != NULL); + + item = pgut_new(pgut_atexit_item); + item->callback = callback; + item->userdata = userdata; + item->next = pgut_atexit_stack; + + pgut_atexit_stack = item; +} + +void +pgut_atexit_pop(pgut_atexit_callback callback, void *userdata) +{ + pgut_atexit_item *item; + pgut_atexit_item **prev; + + for (item = pgut_atexit_stack, prev = &pgut_atexit_stack; + item; + prev = &item->next, item = item->next) + { + if (item->callback == callback && item->userdata == userdata) + { + *prev = item->next; + free(item); + break; + } + } +} + +static void +call_atexit_callbacks(bool fatal) +{ + pgut_atexit_item *item; + + for (item = pgut_atexit_stack; item; item = item->next) + item->callback(fatal, item->userdata); +} + +static void +on_cleanup(void) +{ + in_cleanup = true; + interrupted = false; + call_atexit_callbacks(false); +} + +static void +exit_or_abort(int exitcode) +{ + if (in_cleanup) + { + /* oops, error in cleanup*/ + call_atexit_callbacks(true); + abort(); + } + else + { + /* normal exit */ + exit(exitcode); + } +} + +/* + * Returns the current user name. + */ +static const char * +get_username(void) +{ + const char *ret; + +#ifndef WIN32 + struct passwd *pw; + + pw = getpwuid(geteuid()); + ret = (pw ? pw->pw_name : NULL); +#else + static char username[128]; /* remains after function exit */ + DWORD len = sizeof(username) - 1; + + if (GetUserName(username, &len)) + ret = username; + else + { + _dosmaperr(GetLastError()); + ret = NULL; + } +#endif + + if (ret == NULL) + elog(ERROR, "%s: could not get current user name: %s", + PROGRAM_NAME, strerror(errno)); + return ret; +} + +int +appendStringInfoFile(StringInfo str, FILE *fp) +{ + AssertArg(str != NULL); + AssertArg(fp != NULL); + + for (;;) + { + int rc; + + if (str->maxlen - str->len < 2 && enlargeStringInfo(str, 1024) == 0) + return errno = ENOMEM; + + rc = fread(str->data + str->len, 1, str->maxlen - str->len - 1, fp); + if (rc == 0) + break; + else if (rc > 0) + { + str->len += rc; + str->data[str->len] = '\0'; + } + else if (ferror(fp) && errno != EINTR) + return errno; + } + return 0; +} + +int +appendStringInfoFd(StringInfo str, int fd) +{ + AssertArg(str != NULL); + AssertArg(fd != -1); + + for (;;) + { + int rc; + + if (str->maxlen - str->len < 2 && enlargeStringInfo(str, 1024) == 0) + return errno = ENOMEM; + + rc = read(fd, str->data + str->len, str->maxlen - str->len - 1); + if (rc == 0) + break; + else if (rc > 0) + { + str->len += rc; + str->data[str->len] = '\0'; + } + else if (errno != EINTR) + return errno; + } + return 0; +} + +void * +pgut_malloc(size_t size) +{ + char *ret; + + if ((ret = malloc(size)) == NULL) + elog(ERROR, "could not allocate memory (%lu bytes): %s", + (unsigned long) size, strerror(errno)); + return ret; +} + +void * +pgut_realloc(void *p, size_t size) +{ + char *ret; + + if ((ret = realloc(p, size)) == NULL) + elog(ERROR, "could not re-allocate memory (%lu bytes): %s", + (unsigned long) size, strerror(errno)); + return ret; +} + +char * +pgut_strdup(const char *str) +{ + char *ret; + + if (str == NULL) + return NULL; + + if ((ret = strdup(str)) == NULL) + elog(ERROR, "could not duplicate string \"%s\": %s", + str, strerror(errno)); + return ret; +} + +char * +strdup_with_len(const char *str, size_t len) +{ + char *r; + + if (str == NULL) + return NULL; + + r = pgut_malloc(len + 1); + memcpy(r, str, len); + r[len] = '\0'; + return r; +} + +/* strdup but trim whitespaces at head and tail */ +char * +strdup_trim(const char *str) +{ + size_t len; + + if (str == NULL) + return NULL; + + while (IsSpace(str[0])) { str++; } + len = strlen(str); + while (len > 0 && IsSpace(str[len - 1])) { len--; } + + return strdup_with_len(str, len); +} + +FILE * +pgut_fopen(const char *path, const char *mode, bool missing_ok) +{ + FILE *fp; + + if ((fp = fopen(path, mode)) == NULL) + { + if (missing_ok && errno == ENOENT) + return NULL; + + elog(ERROR, "could not open file \"%s\": %s", + path, strerror(errno)); + } + + return fp; +} + +#ifdef WIN32 +static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout); +#define select select_win32 +#endif + +int +wait_for_socket(int sock, struct timeval *timeout) +{ + fd_set fds; + + FD_ZERO(&fds); + FD_SET(sock, &fds); + return wait_for_sockets(sock + 1, &fds, timeout); +} + +int +wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout) +{ + int i; + + for (;;) + { + i = select(nfds, fds, NULL, NULL, timeout); + if (i < 0) + { + if (interrupted) + elog(ERROR, "interrupted"); + else if (errno != EINTR) + elog(ERROR, "select failed: %s", strerror(errno)); + } + else + return i; + } +} + +#ifndef WIN32 +static void +handle_sigint(SIGNAL_ARGS) +{ + on_interrupt(); +} + +static void +init_cancel_handler(void) +{ + oldhandler = pqsignal(SIGINT, handle_sigint); +} +#else /* WIN32 */ + +/* + * Console control handler for Win32. Note that the control handler will + * execute on a *different thread* than the main one, so we need to do + * proper locking around those structures. + */ +static BOOL WINAPI +consoleHandler(DWORD dwCtrlType) +{ + if (dwCtrlType == CTRL_C_EVENT || + dwCtrlType == CTRL_BREAK_EVENT) + { + EnterCriticalSection(&cancelConnLock); + on_interrupt(); + LeaveCriticalSection(&cancelConnLock); + return TRUE; + } + else + /* Return FALSE for any signals not being handled */ + return FALSE; +} + +static void +init_cancel_handler(void) +{ + InitializeCriticalSection(&cancelConnLock); + + SetConsoleCtrlHandler(consoleHandler, TRUE); +} + +int +sleep(unsigned int seconds) +{ + Sleep(seconds * 1000); + return 0; +} + +int +usleep(unsigned int usec) +{ + Sleep((usec + 999) / 1000); /* rounded up */ + return 0; +} + +#undef select +static int +select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout) +{ + struct timeval remain; + + if (timeout != NULL) + remain = *timeout; + else + { + remain.tv_usec = 0; + remain.tv_sec = LONG_MAX; /* infinite */ + } + + /* sleep only one second because Ctrl+C doesn't interrupt select. */ + while (remain.tv_sec > 0 || remain.tv_usec > 0) + { + int ret; + struct timeval onesec; + + if (remain.tv_sec > 0) + { + onesec.tv_sec = 1; + onesec.tv_usec = 0; + remain.tv_sec -= 1; + } + else + { + onesec.tv_sec = 0; + onesec.tv_usec = remain.tv_usec; + remain.tv_usec = 0; + } + + ret = select(nfds, readfds, writefds, exceptfds, &onesec); + if (ret != 0) + { + /* succeeded or error */ + return ret; + } + else if (interrupted) + { + errno = EINTR; + return 0; + } + } + + return 0; /* timeout */ +} + +#endif /* WIN32 */ diff --git a/src/utils/pgut.h b/src/utils/pgut.h new file mode 100644 index 00000000..fedb99b0 --- /dev/null +++ b/src/utils/pgut.h @@ -0,0 +1,238 @@ +/*------------------------------------------------------------------------- + * + * pgut.h + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PGUT_H +#define PGUT_H + +#include "libpq-fe.h" +#include "pqexpbuffer.h" + +#include +#include + +#include "access/xlogdefs.h" +#include "logger.h" + +#if !defined(C_H) && !defined(__cplusplus) +#ifndef bool +typedef char bool; +#endif +#ifndef true +#define true ((bool) 1) +#endif +#ifndef false +#define false ((bool) 0) +#endif +#endif + +#define INFINITE_STR "INFINITE" + +typedef enum pgut_optsrc +{ + SOURCE_DEFAULT, + SOURCE_FILE_STRICT, + SOURCE_ENV, + SOURCE_FILE, + SOURCE_CMDLINE, + SOURCE_CONST +} pgut_optsrc; + +/* + * type: + * b: bool (true) + * B: bool (false) + * f: pgut_optfn + * i: 32bit signed integer + * u: 32bit unsigned integer + * I: 64bit signed integer + * U: 64bit unsigned integer + * s: string + * t: time_t + */ +typedef struct pgut_option +{ + char type; + uint8 sname; /* short name */ + const char *lname; /* long name */ + void *var; /* pointer to variable */ + pgut_optsrc allowed; /* allowed source */ + pgut_optsrc source; /* actual source */ + int flags; /* option unit */ +} pgut_option; + +typedef void (*pgut_optfn) (pgut_option *opt, const char *arg); +typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); + +/* + * bit values in "flags" of an option + */ +#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ +#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ +#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ +#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ +#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ + +#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ +#define OPTION_UNIT_S 0x20000 /* value is in seconds */ +#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ +#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ + +#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) + +/* + * pgut client variables and functions + */ +extern const char *PROGRAM_NAME; +extern const char *PROGRAM_VERSION; +extern const char *PROGRAM_URL; +extern const char *PROGRAM_EMAIL; + +extern void pgut_help(bool details); + +/* + * pgut framework variables and functions + */ +extern const char *pgut_dbname; +extern const char *host; +extern const char *port; +extern const char *username; +extern bool prompt_password; +extern bool force_password; + +extern bool interrupted; +extern bool in_cleanup; +extern bool in_password; /* User prompts password */ + +extern int pgut_getopt(int argc, char **argv, pgut_option options[]); +extern int pgut_readopt(const char *path, pgut_option options[], int elevel, + bool strict); +extern void pgut_getopt_env(pgut_option options[]); +extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); +extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); + +/* + * Database connections + */ +extern char *pgut_get_conninfo_string(PGconn *conn); +extern PGconn *pgut_connect(const char *dbname); +extern PGconn *pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login); +extern PGconn *pgut_connect_replication(const char *dbname); +extern PGconn *pgut_connect_replication_extended(const char *pghost, const char *pgport, + const char *dbname, const char *pguser); +extern void pgut_disconnect(PGconn *conn); +extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, + const char **params); +extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nParams, + const char **params, bool text_result, bool ok_error); +extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, + const char *query, int nParams, + const char **params, bool text_result); +extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); +extern void pgut_cancel(PGconn* conn); +extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); + +extern const char *pgut_get_host(void); +extern const char *pgut_get_port(void); +extern void pgut_set_host(const char *new_host); +extern void pgut_set_port(const char *new_port); + +/* + * memory allocators + */ +extern void *pgut_malloc(size_t size); +extern void *pgut_realloc(void *p, size_t size); +extern char *pgut_strdup(const char *str); +extern char *strdup_with_len(const char *str, size_t len); +extern char *strdup_trim(const char *str); + +#define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) +#define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) + +/* + * file operations + */ +extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); + +/* + * Assert + */ +#undef Assert +#undef AssertArg +#undef AssertMacro + +#ifdef USE_ASSERT_CHECKING +#define Assert(x) assert(x) +#define AssertArg(x) assert(x) +#define AssertMacro(x) assert(x) +#else +#define Assert(x) ((void) 0) +#define AssertArg(x) ((void) 0) +#define AssertMacro(x) ((void) 0) +#endif + +/* + * StringInfo and string operations + */ +#define STRINGINFO_H + +#define StringInfoData PQExpBufferData +#define StringInfo PQExpBuffer +#define makeStringInfo createPQExpBuffer +#define initStringInfo initPQExpBuffer +#define freeStringInfo destroyPQExpBuffer +#define termStringInfo termPQExpBuffer +#define resetStringInfo resetPQExpBuffer +#define enlargeStringInfo enlargePQExpBuffer +#define printfStringInfo printfPQExpBuffer /* reset + append */ +#define appendStringInfo appendPQExpBuffer +#define appendStringInfoString appendPQExpBufferStr +#define appendStringInfoChar appendPQExpBufferChar +#define appendBinaryStringInfo appendBinaryPQExpBuffer + +extern int appendStringInfoFile(StringInfo str, FILE *fp); +extern int appendStringInfoFd(StringInfo str, int fd); + +extern bool parse_bool(const char *value, bool *result); +extern bool parse_bool_with_len(const char *value, size_t len, bool *result); +extern bool parse_int32(const char *value, int32 *result, int flags); +extern bool parse_uint32(const char *value, uint32 *result, int flags); +extern bool parse_int64(const char *value, int64 *result, int flags); +extern bool parse_uint64(const char *value, uint64 *result, int flags); +extern bool parse_time(const char *value, time_t *result, bool utc_default); +extern bool parse_int(const char *value, int *result, int flags, + const char **hintmsg); +extern bool parse_lsn(const char *value, XLogRecPtr *result); + +extern void convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit); +extern void convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit); + +#define IsSpace(c) (isspace((unsigned char)(c))) +#define IsAlpha(c) (isalpha((unsigned char)(c))) +#define IsAlnum(c) (isalnum((unsigned char)(c))) +#define IsIdentHead(c) (IsAlpha(c) || (c) == '_') +#define IsIdentBody(c) (IsAlnum(c) || (c) == '_') +#define ToLower(c) (tolower((unsigned char)(c))) +#define ToUpper(c) (toupper((unsigned char)(c))) + +/* + * socket operations + */ +extern int wait_for_socket(int sock, struct timeval *timeout); +extern int wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout); + +#ifdef WIN32 +extern int sleep(unsigned int seconds); +extern int usleep(unsigned int usec); +#endif + +#endif /* PGUT_H */ diff --git a/src/utils/thread.c b/src/utils/thread.c new file mode 100644 index 00000000..82c23764 --- /dev/null +++ b/src/utils/thread.c @@ -0,0 +1,102 @@ +/*------------------------------------------------------------------------- + * + * thread.c: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "thread.h" + +pthread_t main_tid = 0; + +#ifdef WIN32 +#include + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} win32_pthread; + +static long mutex_initlock = 0; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + + th->result = th->routine(th->arg); + + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ + +int +pthread_lock(pthread_mutex_t *mp) +{ +#ifdef WIN32 + if (*mp == NULL) + { + while (InterlockedExchange(&mutex_initlock, 1) == 1) + /* loop, another thread own the lock */ ; + if (*mp == NULL) + { + if (pthread_mutex_init(mp, NULL)) + return -1; + } + InterlockedExchange(&mutex_initlock, 0); + } +#endif + return pthread_mutex_lock(mp); +} diff --git a/src/utils/thread.h b/src/utils/thread.h new file mode 100644 index 00000000..06460533 --- /dev/null +++ b/src/utils/thread.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * thread.h: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_THREAD_H +#define PROBACKUP_THREAD_H + +#ifdef WIN32 +#include "postgres_fe.h" +#include "port/pthread-win32.h" + +/* Use native win32 threads on Windows */ +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; + +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +extern int pthread_join(pthread_t th, void **thread_return); +#else +/* Use platform-dependent pthread capability */ +#include +#endif + +extern pthread_t main_tid; + +extern int pthread_lock(pthread_mutex_t *mp); + +#endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c new file mode 100644 index 00000000..bc82e811 --- /dev/null +++ b/src/validate.c @@ -0,0 +1,354 @@ +/*------------------------------------------------------------------------- + * + * validate.c: validate backup files. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); +static void do_validate_instance(void); + +static bool corrupted_backup_found = false; + +typedef struct +{ + parray *files; + bool corrupted; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} validate_files_arg; + +/* + * Validate backup files. + */ +void +pgBackupValidate(pgBackup *backup) +{ + char base_path[MAXPGPATH]; + char path[MAXPGPATH]; + parray *files; + bool corrupted = false; + bool validation_isok = true; + /* arrays with meta info for multi threaded validate */ + pthread_t *threads; + validate_files_arg *threads_args; + int i; + + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE && + backup->status != BACKUP_STATUS_ORPHAN && + backup->status != BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s has status %s. Skip validation.", + base36enc(backup->start_time), status2str(backup->status)); + corrupted_backup_found = true; + return; + } + + if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) + elog(INFO, "Validating backup %s", base36enc(backup->start_time)); + else + elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); + + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->backup_mode != BACKUP_MODE_DIFF_PAGE && + backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && + backup->backup_mode != BACKUP_MODE_DIFF_DELTA) + elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); + + pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + files = dir_read_file_list(base_path, path); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + pg_atomic_clear_flag(&file->lock); + } + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (validate_files_arg *) + palloc(sizeof(validate_files_arg) * num_threads); + + /* Validate files */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->corrupted = false; + /* By default there are some error */ + threads_args[i].ret = 1; + + pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + pthread_join(threads[i], NULL); + if (arg->corrupted) + corrupted = true; + if (arg->ret == 1) + validation_isok = false; + } + if (!validation_isok) + elog(ERROR, "Data files validation failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + /* Update backup status */ + backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; + pgBackupWriteBackupControlFile(backup); + + if (corrupted) + elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); + else + elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); +} + +/* + * Validate files in the backup. + * NOTE: If file is not valid, do not use ERROR log message, + * rather throw a WARNING and set arguments->corrupted = true. + * This is necessary to update backup status. + */ +static void * +pgBackupValidateFiles(void *arg) +{ + int i; + validate_files_arg *arguments = (validate_files_arg *)arg; + pg_crc32 crc; + + for (i = 0; i < parray_num(arguments->files); i++) + { + struct stat st; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + if (interrupted) + elog(ERROR, "Interrupted during validate"); + + /* Validate only regular files */ + if (!S_ISREG(file->mode)) + continue; + /* + * Skip files which has no data, because they + * haven't changed between backups. + */ + if (file->write_size == BYTES_INVALID) + continue; + + /* + * Currently we don't compute checksums for + * cfs_compressed data files, so skip them. + */ + if (file->is_cfs) + continue; + + /* print progress */ + elog(VERBOSE, "Validate files: (%d/%lu) %s", + i + 1, (unsigned long) parray_num(arguments->files), file->path); + + if (stat(file->path, &st) == -1) + { + if (errno == ENOENT) + elog(WARNING, "Backup file \"%s\" is not found", file->path); + else + elog(WARNING, "Cannot stat backup file \"%s\": %s", + file->path, strerror(errno)); + arguments->corrupted = true; + break; + } + + if (file->write_size != st.st_size) + { + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); + arguments->corrupted = true; + break; + } + + crc = pgFileGetCRC(file->path); + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + arguments->corrupted = true; + break; + } + } + + /* Data files validation is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Validate all backups in the backup catalog. + * If --instance option was provided, validate only backups of this instance. + */ +int +do_validate_all(void) +{ + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + xlog_seg_size = get_config_xlog_seg_size(); + + do_validate_instance(); + } + } + else + { + do_validate_instance(); + } + + if (corrupted_backup_found) + { + elog(WARNING, "Some backups are not valid"); + return 1; + } + else + elog(INFO, "All backups are valid"); + + return 0; +} + +/* + * Validate all backups in the given instance of the backup catalog. + */ +static void +do_validate_instance(void) +{ + char *current_backup_id; + int i; + parray *backups; + pgBackup *current_backup = NULL; + + elog(INFO, "Validate backups of the instance '%s'", instance_name); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Examine backups one by one and validate them */ + for (i = 0; i < parray_num(backups); i++) + { + current_backup = (pgBackup *) parray_get(backups, i); + + /* Valiate each backup along with its xlog files. */ + pgBackupValidate(current_backup); + + /* Ensure that the backup has valid list of parent backups */ + if (current_backup->status == BACKUP_STATUS_OK) + { + pgBackup *base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* Validate corresponding WAL files */ + validate_wal(current_backup, arclog_path, 0, + 0, 0, base_full_backup->tli, xlog_seg_size); + } + + /* Mark every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (current_backup->status == BACKUP_STATUS_CORRUPT) + { + int j; + + corrupted_backup_found = true; + current_backup_id = base36enc_dup(current_backup->start_time); + for (j = i - 1; j >= 0; j--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, j); + + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + base36enc(backup->start_time), current_backup_id); + } + } + free(current_backup_id); + } + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); +} diff --git a/tests/Readme.md b/tests/Readme.md new file mode 100644 index 00000000..31dfb656 --- /dev/null +++ b/tests/Readme.md @@ -0,0 +1,24 @@ +[см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) + +``` +Note: For now there are tests only for Linix +``` + + +``` +Check physical correctness of restored instances: + Apply this patch to disable HINT BITS: https://gist.github.com/gsmol/2bb34fd3ba31984369a72cc1c27a36b6 + export PG_PROBACKUP_PARANOIA=ON + +Check archive compression: + export ARCHIVE_COMPRESSION=ON + +Specify path to pg_probackup binary file. By default tests use /pg_probackup/ + export PGPROBACKUPBIN= + +Usage: + pip install testgres + pip install psycopg2 + export PG_CONFIG=/path/to/pg_config + python -m unittest [-v] tests[.specific_module][.class.test] +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..aeeabf2a --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,69 @@ +import unittest + +from . import init_test, option_test, show_test, \ + backup_test, delete_test, restore_test, validate_test, \ + retention_test, ptrack_clean, ptrack_cluster, \ + ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ + ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ + ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ + false_positive, replica, compression, page, ptrack, archive, \ + exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test + + +def load_tests(loader, tests, pattern): + suite = unittest.TestSuite() +# suite.addTests(loader.loadTestsFromModule(auth_test)) + suite.addTests(loader.loadTestsFromModule(archive)) + suite.addTests(loader.loadTestsFromModule(backup_test)) + suite.addTests(loader.loadTestsFromModule(cfs_backup)) +# suite.addTests(loader.loadTestsFromModule(cfs_restore)) +# suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) +# suite.addTests(loader.loadTestsFromModule(logging)) + suite.addTests(loader.loadTestsFromModule(compression)) + suite.addTests(loader.loadTestsFromModule(delete_test)) + suite.addTests(loader.loadTestsFromModule(exclude)) + suite.addTests(loader.loadTestsFromModule(false_positive)) + suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(option_test)) + suite.addTests(loader.loadTestsFromModule(page)) + suite.addTests(loader.loadTestsFromModule(ptrack)) + suite.addTests(loader.loadTestsFromModule(ptrack_clean)) + suite.addTests(loader.loadTestsFromModule(ptrack_cluster)) + suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace)) + suite.addTests(loader.loadTestsFromModule(ptrack_recovery)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) + suite.addTests(loader.loadTestsFromModule(replica)) + suite.addTests(loader.loadTestsFromModule(restore_test)) + suite.addTests(loader.loadTestsFromModule(retention_test)) + suite.addTests(loader.loadTestsFromModule(show_test)) + suite.addTests(loader.loadTestsFromModule(validate_test)) + suite.addTests(loader.loadTestsFromModule(pgpro560)) + suite.addTests(loader.loadTestsFromModule(pgpro589)) + + return suite + +# test_pgpro434_2 unexpected success +# ToDo: +# archive: +# discrepancy of instance`s SYSTEMID and node`s SYSTEMID should lead to archive-push refusal to work +# replica: +# backup should exit with correct error message if some master* option is missing +# --master* options shoukd not work when backuping master +# logging: +# https://jira.postgrespro.ru/browse/PGPRO-584 +# https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md +# ptrack: +# ptrack backup on replica should work correctly +# archive: +# immediate recovery and full recovery +# backward compatibility: +# previous version catalog must be readable by newer version +# incremental chain from previous version can be continued +# backups from previous version can be restored +# 10vanilla_1.3ptrack + +# 10vanilla+ +# 9.6vanilla_1.3ptrack + diff --git a/tests/archive.py b/tests/archive.py new file mode 100644 index 00000000..8b8eb71a --- /dev/null +++ b/tests/archive.py @@ -0,0 +1,833 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script +from datetime import datetime, timedelta +import subprocess +from sys import exit +from time import sleep + + +module_name = 'archive' + + +class ArchiveTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_pgpro434_1(self): + """Description in jira issue PGPRO-434""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector from " + "generate_series(0,100) i") + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-file=verbose"]) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node) + node.slow_start() + + # Recreate backup calagoue + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Make backup + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-file=verbose"]) + node.cleanup() + + # Restore Database + self.restore_node( + backup_dir, 'node', node, + options=["--recovery-target-action=promote"]) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro434_2(self): + """ + Check that timelines are correct. + WAITING PGPRO-1053 for --immediate + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FIRST TIMELINE + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + backup_id = self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select 100501 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + # SECOND TIMELIN + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 " + "from t_heap where id = 100501)")[0][0], + 'data after restore not equal to original data') + + node.safe_psql( + "postgres", + "insert into t_heap select 2 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(100,200) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select 100502 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # THIRD TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print( + node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + node.safe_psql( + "postgres", + "insert into t_heap select 3 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(200,300) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.safe_psql( + "postgres", + "insert into t_heap select 100503 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # FOURTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fourth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # FIFTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fifth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # SIXTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Sixth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 from t_heap where id > 100500)")[0][0], + 'data after restore not equal to original data') + + self.assertEqual( + result, + node.safe_psql( + "postgres", + "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro434_3(self): + """Check pg_stop_backup_timeout, needed backup_timeout""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + archive_script_path = os.path.join(backup_dir, 'archive_script.sh') + with open(archive_script_path, 'w+') as f: + f.write( + archive_script.format( + backup_dir=backup_dir, node_name='node', count_limit=2)) + + st = os.stat(archive_script_path) + os.chmod(archive_script_path, st.st_mode | 0o111) + node.append_conf( + 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( + archive_script_path)) + node.slow_start() + try: + self.backup_node( + backup_dir, 'node', node, + options=[ + "--archive-timeout=60", + "--log-level-file=verbose", + "--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because pg_stop_backup failed to answer.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: pg_stop_backup doesn't answer" in e.message and + "cancel it" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_arhive_push_file_exists(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + file = os.path.join(wals_dir, '000000010000000000000001.gz') + else: + file = os.path.join(wals_dir, '000000010000000000000001') + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content and + 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + os.remove(file) + self.switch_wal_segment(node) + sleep(5) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_arhive_push_file_exists_overwrite(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + file = os.path.join(wals_dir, '000000010000000000000001.gz') + else: + file = os.path.join(wals_dir, '000000010000000000000001') + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content and + 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + self.set_archiving(backup_dir, 'node', node, overwrite=True) + node.reload() + self.switch_wal_segment(node) + sleep(2) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_replica_archive(self): + """ + make node without archiving, take stream backup and + turn it into replica, set replica with archiving, + make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'max_wal_size': '1GB'} + ) + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica, synchronous=True) + + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # ADD INSTANCE 'REPLICA' + + sleep(1) + + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', + replica, backup_type='page', + options=[ + '--archive-timeout=30', '--log-level-file=verbose', + '--master-host=localhost', '--master-db=postgres', + '--master-port={0}'.format(master.port)] + ) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_parallel_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=True) + # ADD INSTANCE REPLICA + self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=20', + '--log-level-file=verbose', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)] + ) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_concurrent_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node( + backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=True) + # ADD INSTANCE REPLICA + # self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + # self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node( + backup_dir, 'master', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog(self): + """Test backup with pg_receivexlog wal delivary method""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + if self.get_version(node) < 100000: + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node') + ], async=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, + node.safe_psql( + "postgres", "SELECT * FROM t_heap" + ), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog_compression_pg10(self): + """Test backup with pg_receivewal compressed wal delivary method""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + if self.get_version(node) < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL 10 for this test') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') + ], async=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, 'node', node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) diff --git a/tests/auth_test.py b/tests/auth_test.py new file mode 100644 index 00000000..fc21a480 --- /dev/null +++ b/tests/auth_test.py @@ -0,0 +1,391 @@ +""" +The Test suite check behavior of pg_probackup utility, if password is required for connection to PostgreSQL instance. + - https://confluence.postgrespro.ru/pages/viewpage.action?pageId=16777522 +""" + +import os +import unittest +import signal +import time + +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import StartNodeException + +module_name = 'auth_test' +skip_test = False + + +try: + from pexpect import * +except ImportError: + skip_test = True + + +class SimpleAuthTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_backup_via_unpriviledged_user(self): + """ + Make node, create unpriviledged user, try to + run a backups without EXECUTE rights on + certain functions + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_start_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_start_backup(text, boolean, boolean) TO backup;") + + time.sleep(1) + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied for function " + "pg_create_restore_point\nquery was: " + "SELECT pg_catalog.pg_create_restore_point($1)", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_create_restore_point(text) TO backup;") + + time.sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_stop_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + if self.get_version(node) < self.version_to_num('10.0'): + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION " + "pg_stop_backup(boolean, boolean) TO backup") + # Do this for ptrack backups + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup") + + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + + node.safe_psql("postgres", "CREATE DATABASE test1") + + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + + node.safe_psql( + "test1", "create table t1 as select generate_series(0,100)") + + node.append_conf("postgresql.auto.conf", "ptrack_enable = 'on'") + node.restart() + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on clearing ptrack_files.") + except ProbackupException as e: + self.assertIn( + "ERROR: must be superuser or replication role to clear ptrack files\n" + "query was: SELECT pg_catalog.pg_ptrack_clear()", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + time.sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on clearing ptrack_files.") + except ProbackupException as e: + self.assertIn( + "ERROR: must be superuser or replication role read ptrack files\n" + "query was: select pg_catalog.pg_ptrack_control_lsn()", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "ALTER ROLE backup REPLICATION") + + time.sleep(1) + + # FULL + self.backup_node( + backup_dir, 'node', node, + options=['-U', 'backup']) + + # PTRACK + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + +class AuthTest(unittest.TestCase): + pb = None + node = None + + @classmethod + def setUpClass(cls): + + super(AuthTest, cls).setUpClass() + + cls.pb = ProbackupTest() + cls.backup_dir = os.path.join(cls.pb.tmp_path, module_name, 'backup') + + cls.node = cls.pb.make_simple_node( + base_dir="{}/node".format(module_name), + set_replication=True, + initdb_params=['--data-checksums', '--auth-host=md5'], + pg_options={ + 'wal_level': 'replica' + } + ) + modify_pg_hba(cls.node) + + cls.pb.init_pb(cls.backup_dir) + cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) + cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) + try: + cls.node.start() + except StartNodeException: + raise unittest.skip("Node hasn't started") + + cls.node.safe_psql("postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; \ + GRANT USAGE ON SCHEMA pg_catalog TO backup; \ + GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_current() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") + cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + + @classmethod + def tearDownClass(cls): + cls.node.cleanup() + cls.pb.del_test_dir(module_name, '') + + @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") + def setUp(self): + self.cmd = ['backup', + '-B', self.backup_dir, + '--instance', self.node.name, + '-h', '127.0.0.1', + '-p', str(self.node.port), + '-U', 'backup', + '-b', 'FULL' + ] + + def tearDown(self): + if "PGPASSWORD" in self.pb.test_env.keys(): + del self.pb.test_env["PGPASSWORD"] + + if "PGPASSWORD" in self.pb.test_env.keys(): + del self.pb.test_env["PGPASSFILE"] + + try: + os.remove(self.pgpass_file) + except OSError: + pass + + def test_empty_password(self): + """ Test case: PGPB_AUTH03 - zero password length """ + try: + self.assertIn("ERROR: no password supplied", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, '\0\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_wrong_password(self): + """ Test case: PGPB_AUTH04 - incorrect password """ + try: + self.assertIn("password authentication failed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'wrong_password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_right_password(self): + """ Test case: PGPB_AUTH01 - correct password """ + try: + self.assertIn("completed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_right_password_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + create_pgpass(self.pgpass_file, line) + try: + self.assertIn("completed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd + ['-W'], 'password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_ctrl_c_event(self): + """ Test case: PGPB_AUTH02 - send interrupt signal """ + try: + run_pb_with_auth([self.pb.probackup_path] + self.cmd, kill=True) + except TIMEOUT: + self.fail("Error: CTRL+C event ignored") + + def test_pgpassfile_env(self): + """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ + path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + create_pgpass(path, line) + self.pb.test_env["PGPASSFILE"] = path + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpass(self): + """ Test case: PGPB_AUTH07 - Create file .pgpass in home dir. """ + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + create_pgpass(self.pgpass_file, line) + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpassword(self): + """ Test case: PGPB_AUTH08 - set environment var PGPASSWORD """ + self.pb.test_env["PGPASSWORD"] = "password" + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpassword_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH09 - Check priority between PGPASSWORD and .pgpass file""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + create_pgpass(self.pgpass_file, line) + self.pb.test_env["PGPASSWORD"] = "password" + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + +def run_pb_with_auth(cmd, password=None, kill=False): + try: + with spawn(" ".join(cmd), encoding='utf-8', timeout=10) as probackup: + result = probackup.expect(u"Password for user .*:", 5) + if kill: + probackup.kill(signal.SIGINT) + elif result == 0: + probackup.sendline(password) + probackup.expect(EOF) + return probackup.before + else: + raise ExceptionPexpect("Other pexpect errors.") + except TIMEOUT: + raise TIMEOUT("Timeout error.") + except ExceptionPexpect: + raise ExceptionPexpect("Pexpect error.") + + +def modify_pg_hba(node): + """ + Description: + Add trust authentication for user postgres. Need for add new role and set grant. + :param node: + :return None: + """ + hba_conf = os.path.join(node.data_dir, "pg_hba.conf") + with open(hba_conf, 'r+') as fio: + data = fio.read() + fio.seek(0) + fio.write('host\tall\tpostgres\t127.0.0.1/0\ttrust\n' + data) + + +def create_pgpass(path, line): + with open(path, 'w') as passfile: + # host:port:db:username:password + passfile.write(line) + os.chmod(path, 0o600) diff --git a/tests/backup_test.py b/tests/backup_test.py new file mode 100644 index 00000000..1fa74643 --- /dev/null +++ b/tests/backup_test.py @@ -0,0 +1,522 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.cfs_helpers import find_by_name + + +module_name = 'backup' + + +class BackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-707 + def test_backup_modes_archive(self): + """standart backup modes with ARCHIVE WAL method""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # postmaster.pid and postmaster.opts shouldn't be copied + excluded = True + db_dir = os.path.join( + backup_dir, "backups", 'node', backup_id, "database") + + for f in os.listdir(db_dir): + if ( + os.path.isfile(os.path.join(db_dir, f)) and + ( + f == "postmaster.pid" or + f == "postmaster.opts" + ) + ): + excluded = False + self.assertEqual(excluded, True) + + # page backup mode + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + # print self.show_pb(node) + show_backup = self.show_pb(backup_dir, 'node')[1] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Check parent backup + self.assertEqual( + backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup['id'])["parent-backup-id"]) + + # ptrack backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + show_backup = self.show_pb(backup_dir, 'node')[2] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PTRACK") + + # Check parent backup + self.assertEqual( + page_backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup['id'])["parent-backup-id"]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_smooth_checkpoint(self): + """full backup with smooth checkpoint""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + options=["-C"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + node.stop() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incremental_backup_without_full(self): + """page-level backup without validated full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], + "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incremental_backup_corrupt_full(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", backup_id, + "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of validation of corrupted backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'\n" in e.message and + "WARNING: Backup file \"{0}\" is not found\n".format( + file) in e.message and + "WARNING: Backup {0} data files are corrupted\n".format( + backup_id) in e.message and + "WARNING: Some backups are not valid\n" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_threads(self): + """ptrack multi thread backup mode""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_threads_stream(self): + """ptrack multi thread backup mode and stream""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_1(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream", '--log-level-file=verbose']) + + # open log file and check + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) + f.close + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', + "Backup Status should be OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_2(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of page " + "corruption in PostgreSQL instance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', + "Backup Status should be ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_tablespace_in_pgdata_pgpro_1376(self): + """PGPRO-1376 """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=( + os.path.join( + node.data_dir, 'somedirectory', '100500')) + ) + + self.create_tblspace_in_node( + node, 'tblspace2', + tblspc_path=(os.path.join(node.data_dir)) + ) + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace tblspace1 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + node.safe_psql( + "postgres", + "create table t_heap2 tablespace tblspace2 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of too many levels " + "of symbolic linking\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Too many levels of symbolic links' in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "drop table t_heap2") + node.safe_psql( + "postgres", + "drop tablespace tblspace2") + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content(node.data_dir) + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap1'::regclass::oid" + ).rstrip() + + list = [] + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file == relfilenode: + path = os.path.join(root, file) + list = list + [path] + + # We expect that relfilenode occures only once + if len(list) > 1: + message = "" + for string in list: + message = message + string + "\n" + self.assertEqual( + 1, 0, + "Following file copied twice by backup:\n {0}".format( + message) + ) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py new file mode 100644 index 00000000..41232032 --- /dev/null +++ b/tests/cfs_backup.py @@ -0,0 +1,1161 @@ +import os +import unittest +import random +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): + # --- Begin --- # + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def setUp(self): + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join( + self.tmp_path, module_name, self.fname, 'backup') + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2', + 'shared_buffers': '200MB' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.start() + + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( + tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not created " + "or it create without compressions" + ) + + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + + # --- Section: Full --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace(self): + """Case: Check fullbackup empty compressed tablespace""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_stream(self): + """Case: Check fullbackup empty compressed tablespace with options stream""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table(self): + """Case: Make full backup after created table in the tablespace""" + if not self.enterprise: + return + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "\n ERROR: {0}\n CMD: {1}".format( + repr(e.message), + repr(self.cmd) + ) + ) + return False + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in {0}".format( + os.path.join(self.backup_dir, 'node', backup_id)) + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table_stream(self): + """ + Case: Make full backup after created table in the tablespace with option --stream + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # --- Section: Incremental from empty tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # --- Section: Incremental from fill tablespace --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,25) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap', tblspace_name) + ) + + full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap') + ) + + page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, options=["-j", "4"]) + self.node.start() + self.assertEqual( + full_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, options=["-j", "4"]) + self.node.start() + self.assertEqual( + page_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments_in_multiple_tablespaces(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + tblspace_name_1 = 'tblspace_name_1' + tblspace_name_2 = 'tblspace_name_2' + + self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True) + self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_1', tblspace_name_1) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_2', tblspace_name_2) + ) + + full_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + full_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_1') + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_2') + ) + + page_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + page_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_1), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_2), + ignore_errors=True) + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, options=["-j", "4"]) + self.node.start() + self.assertEqual( + full_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + full_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_1), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_2), + ignore_errors=True) + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, options=["-j", "4"]) + self.node.start() + self.assertEqual( + page_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + page_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # --- Make backup with not valid data(broken .cfm) --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_cfm_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + os.remove(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_file_pg_compression_from_tablespace_dir(self): + os.remove( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0]) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_data_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + os.remove(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_cfm_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + corrupt_file(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_data_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + corrupt_file(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_file_pg_compression_into_tablespace_dir(self): + + corrupted_file = find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0] + + self.assertTrue( + corrupt_file(corrupted_file), + "ERROR: File is not corrupted or it missing" + ) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + +# # --- End ---# +# @unittest.skipUnless(ProbackupTest.enterprise, 'skip') +# def tearDown(self): +# self.node.cleanup() +# self.del_test_dir(module_name, self.fname) + + +#class CfsBackupEncTest(CfsBackupNoEncTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py new file mode 100644 index 00000000..73553a30 --- /dev/null +++ b/tests/cfs_restore.py @@ -0,0 +1,450 @@ +""" +restore + Syntax: + + pg_probackup restore -B backupdir --instance instance_name + [-D datadir] + [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] + [-j num_threads] [--progress] [-q] [-v] + +""" +import os +import unittest +import shutil + +from .helpers.cfs_helpers import find_by_name +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'cfs_restore' + +tblspace_name = 'cfs_tblspace' +tblspace_name_new = 'cfs_tblspace_new' + + +class CfsRestoreBase(ProbackupTest, unittest.TestCase): + def setUp(self): + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', +# 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.start() + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + self.add_data_in_cluster() + + self.backup_id = None + try: + self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + def add_data_in_cluster(self): + pass + + def tearDown(self): + self.node.cleanup() + self.del_test_dir(module_name, self.fname) + + +class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_empty_tablespace_from_fullbackup(self): + """ + Case: Restore empty tablespace from valid full backup. + """ + self.node.stop(["-m", "immediate"]) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), + "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." + ) + + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not restored or it restored without compressions" + ) + + +class CfsRestoreNoencTest(CfsRestoreBase): + def add_data_in_cluster(self): + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,1e5) i'.format('t1', tblspace_name) + ) + self.table_t1 = self.node.safe_psql( + "postgres", + "SELECT * FROM t1" + ) + + # --- Restore from full backup ---# + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in tablespace dir" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_3_jobs(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + self.node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + self.node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_5_jobs(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + self.node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id, options=['-j', '5']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + self.node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-j", "3", "-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_new_jobs(self): + pass + +# --------------------------------------------------------- # + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_new_jobs(self): + """ + Case: Restore from backup to new location, four jobs + """ + pass + + +#class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() +# +# +#class CfsRestoreEncTest(CfsRestoreNoencTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_validate_backup.py b/tests/cfs_validate_backup.py new file mode 100644 index 00000000..eea6f0e2 --- /dev/null +++ b/tests/cfs_validate_backup.py @@ -0,0 +1,25 @@ +import os +import unittest +import random + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_validate_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): + def setUp(self): + pass + + def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): + pass + + def tearDown(self): + pass + + +#class CfsValidateBackupNoenc(CfsValidateBackupNoenc): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsValidateBackupNoenc).setUp() diff --git a/tests/compression.py b/tests/compression.py new file mode 100644 index 00000000..aa275382 --- /dev/null +++ b/tests/compression.py @@ -0,0 +1,496 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'compression' + + +class CompressionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_compression_stream_zlib(self): + """make archive node, make full and page stream backups, check data correctness in restored instance""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=[ + '--stream', + '--compress-algorithm=zlib']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=[ + '--stream', '--compress-algorithm=zlib', + '--log-level-console=verbose', + '--log-level-file=verbose']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_archive_zlib(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=["--compress-algorithm=zlib"]) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--compress-algorithm=zlib"]) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_stream_pglz(self): + """ + make archive node, make full and page stream backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--compress-algorithm=pglz']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_archive_pglz(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--compress-algorithm=pglz']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_wrong_algorithm(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--compress-algorithm=bla-blah']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because compress-algorithm is invalid.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: invalid compress algorithm value "bla-blah"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delete_test.py b/tests/delete_test.py new file mode 100644 index 00000000..4afb15ae --- /dev/null +++ b/tests/delete_test.py @@ -0,0 +1,203 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from sys import exit + + +module_name = 'delete' + + +class DeleteTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_delete_full_backups(self): + """delete full backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + id_1 = show_backups[0]['id'] + id_2 = show_backups[1]['id'] + id_3 = show_backups[2]['id'] + self.delete_pb(backup_dir, 'node', id_2) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_3) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_increment_page(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_increment_ptrack(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_orphaned_wal_segments(self): + """make archive node, make three full backups, delete second backup without --wal option, then delete orphaned wals via --wal option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + # Check wals + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + original_wal_quantity = len(wals) + + # delete second full backup + self.delete_pb(backup_dir, 'node', backup_2_id) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + # try to delete wals for second backup + self.delete_pb(backup_dir, 'node', options=['--wal']) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # delete first full backup + self.delete_pb(backup_dir, 'node', backup_1_id) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + result = self.delete_pb(backup_dir, 'node', options=['--wal']) + # delete useless wals + self.assertTrue('INFO: removed min WAL segment' in result + and 'INFO: removed max WAL segment' in result) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # Check quantity, it should be lower than original + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + self.assertTrue(original_wal_quantity > len(wals), "Number of wals not changed after 'delete --wal' which is illegal") + + # Delete last backup + self.delete_pb(backup_dir, 'node', backup_3_id, options=['--wal']) + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + self.assertEqual (0, len(wals), "Number of wals should be equal to 0") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py new file mode 100644 index 00000000..40450016 --- /dev/null +++ b/tests/delta.py @@ -0,0 +1,1265 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import QueryException +import subprocess +import time + + +module_name = 'delta' + + +class DeltaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_1(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose" + ] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_2(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose", + "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_3(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10100000) i;" + ) + filepath = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')" + ).rstrip() + + self.backup_node(backup_dir, 'node', node) + + print(os.path.join(node.data_dir, filepath + '.1')) + os.unlink(os.path.join(node.data_dir, filepath + '.1')) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose" + ] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_stream(self): + """ + make archive node, take full and delta stream backups, + restore them and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_archive(self): + """ + make archive node, take full and delta archive backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check delta and data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init( + scale=100, + options=['--tablespace=somedata', '--no-vacuum']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # delta BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream']) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_full(self): + """ + make node, make full and delta stream backups, + restore them and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + # create async connection + conn = self.get_async_connect(port=node.port) + + self.wait(conn) + + acurs = conn.cursor() + acurs.execute("select pg_backend_pid()") + + self.wait(conn) + pid = acurs.fetchall()[0][0] + print(pid) + + gdb = self.gdb_attach(pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + if not gdb.continue_execution_until_running(): + print('Failed gdb continue') + exit(1) + + acurs.execute("VACUUM FULL t_heap") + + if gdb.stopped_in_breakpoint(): + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream'] + ) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take delta backup, + restore database and check it presense + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND DELTA BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exists_in_previous_backup(self): + """ + Make node, take full backup, create table, take page backup, + take delta backup, check that file is no fully copied to delta backup + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + filepath = node.safe_psql( + "postgres", + "SELECT pg_relation_filepath('t_heap')").rstrip() + self.backup_node( + backup_dir, + 'node', + node, + options=["--stream"]) + + # PAGE BACKUP + backup_id = self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + +# if self.paranoia: +# pgdata_page = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream", "--log-level-file=verbose"] + ) +# if self.paranoia: +# pgdata_delta = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) +# self.compare_pgdata( +# pgdata_page, pgdata_delta) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_delta(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new" + ) + + # DELTA BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ), + "--recovery-target-action=promote" + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_delta(self): + """ + Make node, take full backup, create database, + take delta backup, alter database tablespace location, + take delta backup restore last delta backup. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql( + "postgres", + "create database db1 tablespace = 'somedata'") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter database db1 set tablespace somedata_new" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_delete(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_1(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream", "--log-level-file=verbose"]) + + # open log file and check + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) + f.close + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', + "Backup Status should be OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_2(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of page " + "corruption in PostgreSQL instance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', + "Backup Status should be ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/exclude.py b/tests/exclude.py new file mode 100644 index 00000000..48b7889c --- /dev/null +++ b/tests/exclude.py @@ -0,0 +1,164 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'exclude' + + +class ExcludeTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_exclude_temp_tables(self): + """ + make node without archiving, create temp table, take full backup, + check that temp table not present in backup catalogue + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create temp table test as " + "select generate_series(0,50050000)::text") + conn.commit() + + temp_schema_name = conn.execute( + "SELECT nspname FROM pg_namespace " + "WHERE oid = pg_my_temp_schema()")[0][0] + conn.commit() + + temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( + "pg_", "") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + heap_path = conn.execute( + "select pg_relation_filepath('test')")[0][0] + conn.commit() + + index_path = conn.execute( + "select pg_relation_filepath('test_idx')")[0][0] + conn.commit() + + heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] + conn.commit() + + toast_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] + conn.commit() + + toast_idx_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, + "pg_toast_" + str(heap_oid) + "_index"))[0][0] + conn.commit() + + temp_table_filename = os.path.basename(heap_path) + temp_idx_filename = os.path.basename(index_path) + temp_toast_filename = os.path.basename(toast_path) + temp_idx_toast_filename = os.path.basename(toast_idx_path) + + self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file in [ + temp_table_filename, temp_table_filename + ".1", + temp_idx_filename, + temp_idx_filename + ".1", + temp_toast_filename, + temp_toast_filename + ".1", + temp_idx_toast_filename, + temp_idx_toast_filename + ".1" + ]: + self.assertEqual( + 1, 0, + "Found temp table file in backup catalogue.\n " + "Filepath: {0}".format(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exclude_unlogged_tables_1(self): + """ + make node without archiving, create unlogged table, take full backup, + alter table to unlogged, take ptrack backup, restore ptrack backup, + check that PGDATA`s are physically the same + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + "shared_buffers": "10MB", + "fsync": "off", + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create unlogged table test as " + "select generate_series(0,5005000)::text") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + node.safe_psql('postgres', "alter table test set logged") + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out new file mode 100644 index 00000000..35f58406 --- /dev/null +++ b/tests/expected/option_help.out @@ -0,0 +1,95 @@ + +pg_probackup - utility to manage backup/recovery of PostgreSQL database. + + pg_probackup help [COMMAND] + + pg_probackup version + + pg_probackup init -B backup-path + + pg_probackup set-config -B backup-dir --instance=instance_name + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup show-config -B backup-dir --instance=instance_name + [--format=format] + + pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + [-C] [--stream [-S slot-name]] [--backup-pg-log] + [-j num-threads] [--archive-timeout=archive-timeout] + [--progress] + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--delete-expired] [--delete-wal] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--compress] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup restore -B backup-dir --instance=instance_name + [-D pgdata-dir] [-i backup-id] [--progress] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--timeline=timeline] [-T OLDDIR=NEWDIR] + [--immediate] [--recovery-target-name=target-name] + [--recovery-target-action=pause|promote|shutdown] + [--restore-as-replica] + [--no-validate] + + pg_probackup validate -B backup-dir [--instance=instance_name] + [-i backup-id] [--progress] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--recovery-target-name=target-name] + [--timeline=timeline] + + pg_probackup show -B backup-dir + [--instance=instance_name [-i backup-id]] + [--format=format] + + pg_probackup delete -B backup-dir --instance=instance_name + [--wal] [-i backup-id | --expired] + + pg_probackup merge -B backup-dir --instance=instance_name + -i backup-id + + pg_probackup add-instance -B backup-dir -D pgdata-dir + --instance=instance_name + + pg_probackup del-instance -B backup-dir + --instance=instance_name + + pg_probackup archive-push -B backup-dir --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + [--compress [--compress-level=compress-level]] + [--overwrite] + + pg_probackup archive-get -B backup-dir --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + +Read the website for details. +Report bugs to . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out new file mode 100644 index 00000000..35e212c3 --- /dev/null +++ b/tests/expected/option_version.out @@ -0,0 +1 @@ +pg_probackup 2.0.18 \ No newline at end of file diff --git a/tests/false_positive.py b/tests/false_positive.py new file mode 100644 index 00000000..1884159b --- /dev/null +++ b/tests/false_positive.py @@ -0,0 +1,333 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + + +module_name = 'false_positive' + + +class FalsePositive(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_validate_wal_lost_segment(self): + """Loose segment located between backups. ExpectedFailure. This is BUG """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(backup_dir, "wal", 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = map(int, wals) + os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) + + # We just lost a wal segment and know nothing about it + self.backup_node(backup_dir, 'node', node) + self.assertTrue( + 'validation completed successfully' in self.validate_pb( + backup_dir, 'node')) + ######## + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.expectedFailure + # Need to force validation of ancestor-chain + def test_incremental_backup_corrupt_full_1(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", + backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + self.assertFalse( + True, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_ptrack_concurrent_get_and_clear_1(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('make_pagemap_from_ptrack') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + tablespace_oid = node.safe_psql( + "postgres", + "select oid from pg_tablespace where spcname = 'pg_default'").rstrip() + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap'::regclass::oid").rstrip() + + node.safe_psql( + "postgres", + "SELECT pg_ptrack_get_and_clear({0}, {1})".format( + tablespace_oid, relfilenode)) + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_ptrack_concurrent_get_and_clear_2(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('pthread_create') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + tablespace_oid = node.safe_psql( + "postgres", + "select oid from pg_tablespace " + "where spcname = 'pg_default'").rstrip() + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap'::regclass::oid").rstrip() + + node.safe_psql( + "postgres", + "SELECT pg_ptrack_get_and_clear({0}, {1})".format( + tablespace_oid, relfilenode)) + + gdb._execute("delete breakpoints") + gdb.continue_execution_until_exit() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup ptrack_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_multiple_delete(self): + """delete multiple backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + self.delete_pb(backup_dir, 'node', options= + ["-i {0}".format(backup_1_id), "-i {0}".format(backup_2_id), "-i {0}".format(backup_3_id)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..ac64c423 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,2 @@ +__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] +#from . import * \ No newline at end of file diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py new file mode 100644 index 00000000..67e2b331 --- /dev/null +++ b/tests/helpers/cfs_helpers.py @@ -0,0 +1,91 @@ +import os +import re +import random +import string + + +def find_by_extensions(dirs=None, extensions=None): + """ + find_by_extensions(['path1','path2'],['.txt','.log']) + :return: + Return list of files include full path by file extensions + """ + files = [] + new_dirs = [] + + if dirs is not None and extensions is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.splitext(d)[1] in extensions: + files.append(d) + + if new_dirs: + files.extend(find_by_extensions(new_dirs, extensions)) + + return files + + +def find_by_pattern(dirs=None, pattern=None): + """ + find_by_pattern(['path1','path2'],'^.*/*.txt') + :return: + Return list of files include full path by pattern + """ + files = [] + new_dirs = [] + + if dirs is not None and pattern is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if re.match(pattern,d): + files.append(d) + + if new_dirs: + files.extend(find_by_pattern(new_dirs, pattern)) + + return files + + +def find_by_name(dirs=None, filename=None): + files = [] + new_dirs = [] + + if dirs is not None and filename is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.basename(d) in filename: + files.append(d) + + if new_dirs: + files.extend(find_by_name(new_dirs, filename)) + + return files + + +def corrupt_file(filename): + file_size = None + try: + file_size = os.path.getsize(filename) + except OSError: + return False + + try: + with open(filename, "rb+") as f: + f.seek(random.randint(int(0.1*file_size),int(0.8*file_size))) + f.write(random_string(0.1*file_size)) + f.close() + except OSError: + return False + + return True + + +def random_string(n): + a = string.ascii_letters + string.digits + return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py new file mode 100644 index 00000000..0d04d898 --- /dev/null +++ b/tests/helpers/ptrack_helpers.py @@ -0,0 +1,1300 @@ +# you need os for unittest to work +import os +from sys import exit, argv, version_info +import subprocess +import shutil +import six +import testgres +import hashlib +import re +import pwd +import select +import psycopg2 +from time import sleep +import re +import json + +idx_ptrack = { + 't_heap': { + 'type': 'heap' + }, + 't_btree': { + 'type': 'btree', + 'column': 'text', + 'relation': 't_heap' + }, + 't_seq': { + 'type': 'seq', + 'column': 't_seq', + 'relation': 't_heap' + }, + 't_spgist': { + 'type': 'spgist', + 'column': 'text', + 'relation': 't_heap' + }, + 't_brin': { + 'type': 'brin', + 'column': 'text', + 'relation': 't_heap' + }, + 't_gist': { + 'type': 'gist', + 'column': 'tsvector', + 'relation': 't_heap' + }, + 't_gin': { + 'type': 'gin', + 'column': 'tsvector', + 'relation': 't_heap' + }, +} + +archive_script = """ +#!/bin/bash +count=$(ls {backup_dir}/test00* | wc -l) +if [ $count -ge {count_limit} ] +then + exit 1 +else + cp $1 {backup_dir}/wal/{node_name}/$2 + count=$((count+1)) + touch {backup_dir}/test00$count + exit 0 +fi +""" +warning = """ +Wrong splint in show_pb +Original Header: +{header} +Original Body: +{body} +Splitted Header +{header_split} +Splitted Body +{body_split} +""" + + +def dir_files(base_dir): + out_list = [] + for dir_name, subdir_list, file_list in os.walk(base_dir): + if dir_name != base_dir: + out_list.append(os.path.relpath(dir_name, base_dir)) + for fname in file_list: + out_list.append( + os.path.relpath(os.path.join( + dir_name, fname), base_dir) + ) + out_list.sort() + return out_list + + +def is_enterprise(): + # pg_config --help + p = subprocess.Popen( + [os.environ['PG_CONFIG'], '--help'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if b'postgrespro.ru' in p.communicate()[0]: + return True + else: + return False + + +class ProbackupException(Exception): + def __init__(self, message, cmd): + self.message = message + self.cmd = cmd + + def __str__(self): + return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) + + +def slow_start(self, replica=False): + + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # raise_operational_error=False) + + self.start() + if not replica: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + else: + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + +# while True: +# try: +# self.poll_query_until( +# "postgres", +# "SELECT pg_is_in_recovery()") +# break +# except ProbackupException as e: +# continue + + +class ProbackupTest(object): + # Class attributes + enterprise = is_enterprise() + + def __init__(self, *args, **kwargs): + super(ProbackupTest, self).__init__(*args, **kwargs) + if '-v' in argv or '--verbose' in argv: + self.verbose = True + else: + self.verbose = False + + self.test_env = os.environ.copy() + envs_list = [ + "LANGUAGE", + "LC_ALL", + "PGCONNECT_TIMEOUT", + "PGDATA", + "PGDATABASE", + "PGHOSTADDR", + "PGREQUIRESSL", + "PGSERVICE", + "PGSSLMODE", + "PGUSER", + "PGPORT", + "PGHOST" + ] + + for e in envs_list: + try: + del self.test_env[e] + except: + pass + + self.test_env["LC_MESSAGES"] = "C" + self.test_env["LC_TIME"] = "C" + + self.paranoia = False + if 'PG_PROBACKUP_PARANOIA' in self.test_env: + if self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON': + self.paranoia = True + + self.archive_compress = False + if 'ARCHIVE_COMPRESSION' in self.test_env: + if self.test_env['ARCHIVE_COMPRESSION'] == 'ON': + self.archive_compress = True + try: + testgres.configure_testgres( + cache_initdb=False, + cached_initdb_dir=False, + cache_pg_config=False, + node_cleanup_full=False) + except: + pass + + self.helpers_path = os.path.dirname(os.path.realpath(__file__)) + self.dir_path = os.path.abspath( + os.path.join(self.helpers_path, os.pardir) + ) + self.tmp_path = os.path.abspath( + os.path.join(self.dir_path, 'tmp_dirs') + ) + try: + os.makedirs(os.path.join(self.dir_path, 'tmp_dirs')) + except: + pass + + self.user = self.get_username() + self.probackup_path = None + if "PGPROBACKUPBIN" in self.test_env: + if ( + os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and + os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) + ): + self.probackup_path = self.test_env["PGPROBACKUPBIN"] + else: + if self.verbose: + print('PGPROBINDIR is not an executable file') + if not self.probackup_path: + self.probackup_path = os.path.abspath(os.path.join( + self.dir_path, "../pg_probackup")) + + def make_simple_node( + self, + base_dir=None, + set_replication=False, + initdb_params=[], + pg_options={}): + + real_base_dir = os.path.join(self.tmp_path, base_dir) + shutil.rmtree(real_base_dir, ignore_errors=True) + os.makedirs(real_base_dir) + + node = testgres.get_new_node('test', base_dir=real_base_dir) + # bound method slow_start() to 'node' class instance + node.slow_start = slow_start.__get__(node) + node.should_rm_dirs = True + node.init( + initdb_params=initdb_params, allow_streaming=set_replication) + + # Sane default parameters + node.append_conf("postgresql.auto.conf", "max_connections = 100") + node.append_conf("postgresql.auto.conf", "shared_buffers = 10MB") + node.append_conf("postgresql.auto.conf", "fsync = on") + node.append_conf("postgresql.auto.conf", "wal_level = logical") + node.append_conf("postgresql.auto.conf", "hot_standby = 'off'") + + node.append_conf( + "postgresql.auto.conf", "log_line_prefix = '%t [%p]: [%l-1] '") + node.append_conf("postgresql.auto.conf", "log_statement = none") + node.append_conf("postgresql.auto.conf", "log_duration = on") + node.append_conf( + "postgresql.auto.conf", "log_min_duration_statement = 0") + node.append_conf("postgresql.auto.conf", "log_connections = on") + node.append_conf("postgresql.auto.conf", "log_disconnections = on") + + # Apply given parameters + for key, value in six.iteritems(pg_options): + node.append_conf("postgresql.auto.conf", "%s = %s" % (key, value)) + + # Allow replication in pg_hba.conf + if set_replication: + node.append_conf( + "pg_hba.conf", + "local replication all trust\n") + node.append_conf( + "postgresql.auto.conf", + "max_wal_senders = 10") + + return node + + def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): + res = node.execute( + "postgres", + "select exists" + " (select 1 from pg_tablespace where spcname = '{0}')".format( + tblspc_name) + ) + # Check that tablespace with name 'tblspc_name' do not exists already + self.assertFalse( + res[0][0], + 'Tablespace "{0}" already exists'.format(tblspc_name) + ) + + if not tblspc_path: + tblspc_path = os.path.join( + node.base_dir, '{0}'.format(tblspc_name)) + cmd = "CREATE TABLESPACE {0} LOCATION '{1}'".format( + tblspc_name, tblspc_path) + if cfs: + cmd += " with (compression=true)" + + if not os.path.exists(tblspc_path): + os.makedirs(tblspc_path) + res = node.safe_psql("postgres", cmd) + # Check that tablespace was successfully created + # self.assertEqual( + # res[0], 0, + # 'Failed to create tablespace with cmd: {0}'.format(cmd)) + + def get_tblspace_path(self, node, tblspc_name): + return os.path.join(node.base_dir, tblspc_name) + + def get_fork_size(self, node, fork_name): + return node.execute( + "postgres", + "select pg_relation_size('{0}')/8192".format(fork_name))[0][0] + + def get_fork_path(self, node, fork_name): + return os.path.join( + node.base_dir, 'data', node.execute( + "postgres", + "select pg_relation_filepath('{0}')".format( + fork_name))[0][0] + ) + + def get_md5_per_page_for_fork(self, file, size_in_pages): + pages_per_segment = {} + md5_per_page = {} + nsegments = size_in_pages/131072 + if size_in_pages % 131072 != 0: + nsegments = nsegments + 1 + + size = size_in_pages + for segment_number in range(nsegments): + if size - 131072 > 0: + pages_per_segment[segment_number] = 131072 + else: + pages_per_segment[segment_number] = size + size = size - 131072 + + for segment_number in range(nsegments): + offset = 0 + if segment_number == 0: + file_desc = os.open(file, os.O_RDONLY) + start_page = 0 + end_page = pages_per_segment[segment_number] + else: + file_desc = os.open( + file+".{0}".format(segment_number), os.O_RDONLY + ) + start_page = max(md5_per_page)+1 + end_page = end_page + pages_per_segment[segment_number] + + for page in range(start_page, end_page): + md5_per_page[page] = hashlib.md5( + os.read(file_desc, 8192)).hexdigest() + offset += 8192 + os.lseek(file_desc, offset, 0) + os.close(file_desc) + + return md5_per_page + + def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): + + if self.get_pgpro_edition(node) == 'enterprise': + header_size = 48 + else: + header_size = 24 + ptrack_bits_for_fork = [] + + page_body_size = 8192-header_size + byte_size = os.path.getsize(file + '_ptrack') + npages = byte_size/8192 + if byte_size % 8192 != 0: + print('Ptrack page is not 8k aligned') + sys.exit(1) + + file = os.open(file + '_ptrack', os.O_RDONLY) + + for page in range(npages): + offset = 8192*page+header_size + os.lseek(file, offset, 0) + lots_of_bytes = os.read(file, page_body_size) + byte_list = [ + lots_of_bytes[i:i+1] for i in range(len(lots_of_bytes)) + ] + for byte in byte_list: + # byte_inverted = bin(int(byte, base=16))[2:][::-1] + # bits = (byte >> x) & 1 for x in range(7, -1, -1) + byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1] + for bit in byte_inverted: + # if len(ptrack_bits_for_fork) < size: + ptrack_bits_for_fork.append(int(bit)) + + os.close(file) + return ptrack_bits_for_fork + + def check_ptrack_sanity(self, idx_dict): + success = True + if idx_dict['new_size'] > idx_dict['old_size']: + size = idx_dict['new_size'] + else: + size = idx_dict['old_size'] + for PageNum in range(size): + if PageNum not in idx_dict['old_pages']: + # Page was not present before, meaning that relation got bigger + # Ptrack should be equal to 1 + if idx_dict['ptrack'][PageNum] != 1: + if self.verbose: + print( + 'Page Number {0} of type {1} was added,' + ' but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + # print(idx_dict) + success = False + continue + if PageNum not in idx_dict['new_pages']: + # Page is not present now, meaning that relation got smaller + # Ptrack should be equal to 0, + # We are not freaking out about false positive stuff + if idx_dict['ptrack'][PageNum] != 0: + if self.verbose: + print( + 'Page Number {0} of type {1} was deleted,' + ' but ptrack value is {2}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + continue + + # Ok, all pages in new_pages that do not have + # corresponding page in old_pages are been dealt with. + # We can now safely proceed to comparing old and new pages + if idx_dict['new_pages'][ + PageNum] != idx_dict['old_pages'][PageNum]: + # Page has been changed, + # meaning that ptrack should be equal to 1 + if idx_dict['ptrack'][PageNum] != 1: + if self.verbose: + print( + 'Page Number {0} of type {1} was changed,' + ' but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + print( + "\n Old checksumm: {0}\n" + " New checksumm: {1}".format( + idx_dict['old_pages'][PageNum], + idx_dict['new_pages'][PageNum]) + ) + + if PageNum == 0 and idx_dict['type'] == 'spgist': + if self.verbose: + print( + 'SPGIST is a special snowflake, so don`t ' + 'fret about losing ptrack for blknum 0' + ) + continue + success = False + else: + # Page has not been changed, + # meaning that ptrack should be equal to 0 + if idx_dict['ptrack'][PageNum] != 0: + if self.verbose: + print( + 'Page Number {0} of type {1} was not changed,' + ' but ptrack value is {2}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum] + ) + ) + + self.assertTrue( + success, 'Ptrack does not correspond to state' + ' of its own pages.\n Gory Details: \n{0}'.format( + idx_dict['type'], idx_dict + ) + ) + + def check_ptrack_recovery(self, idx_dict): + size = idx_dict['size'] + for PageNum in range(size): + if idx_dict['ptrack'][PageNum] != 1: + self.assertTrue( + False, + 'Recovery for Page Number {0} of Type {1}' + ' was conducted, but ptrack value is {2}.' + ' THIS IS BAD\n IDX_DICT: {3}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum], + idx_dict + ) + ) + + def check_ptrack_clean(self, idx_dict, size): + for PageNum in range(size): + if idx_dict['ptrack'][PageNum] != 0: + self.assertTrue( + False, + 'Ptrack for Page Number {0} of Type {1}' + ' should be clean, but ptrack value is {2}.' + '\n THIS IS BAD\n IDX_DICT: {3}'.format( + PageNum, + idx_dict['type'], + idx_dict['ptrack'][PageNum], + idx_dict + ) + ) + + def run_pb(self, command, async=False, gdb=False): + try: + self.cmd = [' '.join(map(str, [self.probackup_path] + command))] + if self.verbose: + print(self.cmd) + if gdb: + return GDBobj([self.probackup_path] + command, self.verbose) + if async: + return subprocess.Popen( + self.cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + self.output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") + if command[0] == 'backup': + # return backup ID + for line in self.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + else: + return self.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode("utf-8"), self.cmd) + + def run_binary(self, command, async=False): + if self.verbose: + print([' '.join(map(str, command))]) + try: + if async: + return subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + self.output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") + return self.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode("utf-8"), command) + + def init_pb(self, backup_dir): + + shutil.rmtree(backup_dir, ignore_errors=True) + return self.run_pb([ + "init", + "-B", backup_dir + ]) + + def add_instance(self, backup_dir, instance, node): + + return self.run_pb([ + "add-instance", + "--instance={0}".format(instance), + "-B", backup_dir, + "-D", node.data_dir + ]) + + def del_instance(self, backup_dir, instance): + + return self.run_pb([ + "del-instance", + "--instance={0}".format(instance), + "-B", backup_dir + ]) + + def clean_pb(self, backup_dir): + shutil.rmtree(backup_dir, ignore_errors=True) + + def backup_node( + self, backup_dir, instance, node, data_dir=False, + backup_type="full", options=[], async=False, gdb=False + ): + if not node and not data_dir: + print('You must provide ether node or data_dir for backup') + exit(1) + + if node: + pgdata = node.data_dir + + if data_dir: + pgdata = data_dir + + cmd_list = [ + "backup", + "-B", backup_dir, + # "-D", pgdata, + "-p", "%i" % node.port, + "-d", "postgres", + "--instance={0}".format(instance) + ] + if backup_type: + cmd_list += ["-b", backup_type] + + return self.run_pb(cmd_list + options, async, gdb) + + def merge_backup(self, backup_dir, instance, backup_id): + cmd_list = [ + "merge", + "-B", backup_dir, + "--instance={0}".format(instance), + "-i", backup_id + ] + + return self.run_pb(cmd_list) + + def restore_node( + self, backup_dir, instance, node=False, + data_dir=None, backup_id=None, options=[] + ): + if data_dir is None: + data_dir = node.data_dir + + cmd_list = [ + "restore", + "-B", backup_dir, + "-D", data_dir, + "--instance={0}".format(instance) + ] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def show_pb( + self, backup_dir, instance=None, backup_id=None, + options=[], as_text=False, as_json=True + ): + + backup_list = [] + specific_record = {} + cmd_list = [ + "show", + "-B", backup_dir, + ] + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if backup_id: + cmd_list += ["-i", backup_id] + + if as_json: + cmd_list += ["--format=json"] + + if as_text: + # You should print it when calling as_text=true + return self.run_pb(cmd_list + options) + + # get show result as list of lines + if as_json: + data = json.loads(self.run_pb(cmd_list + options)) + # print(data) + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: + continue + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) + return backup_list + else: + show_splitted = self.run_pb(cmd_list + options).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(" +", header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(" +", backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(" = ")[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + return specific_record + + def validate_pb( + self, backup_dir, instance=None, + backup_id=None, options=[] + ): + + cmd_list = [ + "validate", + "-B", backup_dir + ] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def delete_pb(self, backup_dir, instance, backup_id=None, options=[]): + cmd_list = [ + "delete", + "-B", backup_dir + ] + + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def delete_expired(self, backup_dir, instance, options=[]): + cmd_list = [ + "delete", "--expired", "--wal", + "-B", backup_dir, + "--instance={0}".format(instance) + ] + return self.run_pb(cmd_list + options) + + def show_config(self, backup_dir, instance): + out_dict = {} + cmd_list = [ + "show-config", + "-B", backup_dir, + "--instance={0}".format(instance) + ] + res = self.run_pb(cmd_list).splitlines() + for line in res: + if not line.startswith('#'): + name, var = line.partition(" = ")[::2] + out_dict[name] = var + return out_dict + + def get_recovery_conf(self, node): + out_dict = {} + with open( + os.path.join(node.data_dir, "recovery.conf"), "r" + ) as recovery_conf: + for line in recovery_conf: + try: + key, value = line.split("=") + except: + continue + out_dict[key.strip()] = value.strip(" '").replace("'\n", "") + return out_dict + + def set_archiving( + self, backup_dir, instance, node, replica=False, overwrite=False): + + if replica: + archive_mode = 'always' + node.append_conf('postgresql.auto.conf', 'hot_standby = on') + else: + archive_mode = 'on' + + # node.append_conf( + # "postgresql.auto.conf", + # "wal_level = archive" + # ) + node.append_conf( + "postgresql.auto.conf", + "archive_mode = {0}".format(archive_mode) + ) + archive_command = "{0} archive-push -B {1} --instance={2} ".format( + self.probackup_path, backup_dir, instance) + + if os.name == 'posix': + if self.archive_compress: + archive_command = archive_command + "--compress " + + if overwrite: + archive_command = archive_command + "--overwrite " + + archive_command = archive_command + "--wal-file-path %p --wal-file-name %f" + + node.append_conf( + "postgresql.auto.conf", + "archive_command = '{0}'".format( + archive_command)) + # elif os.name == 'nt': + # node.append_conf( + # "postgresql.auto.conf", + # "archive_command = 'copy %p {0}\\%f'".format(archive_dir) + # ) + + def set_replica( + self, master, replica, + replica_name='replica', + synchronous=False + ): + replica.append_conf( + "postgresql.auto.conf", "port = {0}".format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf( + "recovery.conf", + "primary_conninfo = 'user={0} port={1} application_name={2}" + " sslmode=prefer sslcompression=1'".format( + self.user, master.port, replica_name) + ) + if synchronous: + master.append_conf( + "postgresql.auto.conf", + "synchronous_standby_names='{0}'".format(replica_name) + ) + master.append_conf( + 'postgresql.auto.conf', + "synchronous_commit='remote_apply'" + ) + master.reload() + + def wrong_wal_clean(self, node, wal_size): + wals_dir = os.path.join(self.backup_dir(node), "wal") + wals = [ + f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) + ] + wals.sort() + file_path = os.path.join(wals_dir, wals[-1]) + if os.path.getsize(file_path) != wal_size: + os.remove(file_path) + + def guc_wal_segment_size(self, node): + var = node.execute( + "postgres", + "select setting from pg_settings where name = 'wal_segment_size'" + ) + return int(var[0][0]) * self.guc_wal_block_size(node) + + def guc_wal_block_size(self, node): + var = node.execute( + "postgres", + "select setting from pg_settings where name = 'wal_block_size'" + ) + return int(var[0][0]) + + def get_pgpro_edition(self, node): + if node.execute( + "postgres", + "select exists (select 1 from" + " pg_proc where proname = 'pgpro_edition')" + )[0][0]: + var = node.execute("postgres", "select pgpro_edition()") + return str(var[0][0]) + else: + return False + + def get_username(self): + """ Returns current user name """ + return pwd.getpwuid(os.getuid())[0] + + def version_to_num(self, version): + if not version: + return 0 + parts = version.split(".") + while len(parts) < 3: + parts.append("0") + num = 0 + for part in parts: + num = num * 100 + int(re.sub("[^\d]", "", part)) + return num + + def switch_wal_segment(self, node): + """ + Execute pg_switch_wal/xlog() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + if self.version_to_num( + node.safe_psql("postgres", "show server_version") + ) >= self.version_to_num('10.0'): + node.safe_psql("postgres", "select pg_switch_wal()") + else: + node.safe_psql("postgres", "select pg_switch_xlog()") + else: + if self.version_to_num( + node.execute("show server_version")[0][0] + ) >= self.version_to_num('10.0'): + node.execute("select pg_switch_wal()") + else: + node.execute("select pg_switch_xlog()") + sleep(1) + + def get_version(self, node): + return self.version_to_num( + testgres.get_pg_config()["VERSION"].split(" ")[1]) + + def get_bin_path(self, binary): + return testgres.get_bin_path(binary) + + def del_test_dir(self, module_name, fname): + """ Del testdir and optimistically try to del module dir""" + try: + testgres.clean_all() + except: + pass + + shutil.rmtree( + os.path.join( + self.tmp_path, + module_name, + fname + ), + ignore_errors=True + ) + try: + os.rmdir(os.path.join(self.tmp_path, module_name)) + except: + pass + + def pgdata_content(self, directory, ignore_ptrack=True): + """ return dict with directory content. " + " TAKE IT AFTER CHECKPOINT or BACKUP""" + dirs_to_ignore = [ + 'pg_xlog', 'pg_wal', 'pg_log', + 'pg_stat_tmp', 'pg_subtrans', 'pg_notify' + ] + files_to_ignore = [ + 'postmaster.pid', 'postmaster.opts', + 'pg_internal.init', 'postgresql.auto.conf', + 'backup_label', 'tablespace_map', 'recovery.conf', + 'ptrack_control', 'ptrack_init', 'pg_control' + ] +# suffixes_to_ignore = ( +# '_ptrack' +# ) + directory_dict = {} + directory_dict['pgdata'] = directory + directory_dict['files'] = {} + for root, dirs, files in os.walk(directory, followlinks=True): + dirs[:] = [d for d in dirs if d not in dirs_to_ignore] + for file in files: + if ( + file in files_to_ignore or + (ignore_ptrack and file.endswith('_ptrack')) + ): + continue + + file_fullpath = os.path.join(root, file) + file_relpath = os.path.relpath(file_fullpath, directory) + directory_dict['files'][file_relpath] = {'is_datafile': False} + directory_dict['files'][file_relpath]['md5'] = hashlib.md5( + open(file_fullpath, 'rb').read()).hexdigest() + + if file.isdigit(): + directory_dict['files'][file_relpath]['is_datafile'] = True + size_in_pages = os.path.getsize(file_fullpath)/8192 + directory_dict['files'][file_relpath][ + 'md5_per_page'] = self.get_md5_per_page_for_fork( + file_fullpath, size_in_pages + ) + + return directory_dict + + def compare_pgdata(self, original_pgdata, restored_pgdata): + """ return dict with directory content. DO IT BEFORE RECOVERY""" + fail = False + error_message = 'Restored PGDATA is not equal to original!\n' + for file in restored_pgdata['files']: + # File is present in RESTORED PGDATA + # but not present in ORIGINAL + # only backup_label is allowed + if file not in original_pgdata['files']: + fail = True + error_message += '\nFile is not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], file)) + + for file in original_pgdata['files']: + if file in restored_pgdata['files']: + + if ( + original_pgdata['files'][file]['md5'] != + restored_pgdata['files'][file]['md5'] + ): + fail = True + error_message += ( + '\nFile Checksumm mismatch.\n' + 'File_old: {0}\nChecksumm_old: {1}\n' + 'File_new: {2}\nChecksumm_new: {3}\n').format( + os.path.join(original_pgdata['pgdata'], file), + original_pgdata['files'][file]['md5'], + os.path.join(restored_pgdata['pgdata'], file), + restored_pgdata['files'][file]['md5'] + ) + + if original_pgdata['files'][file]['is_datafile']: + for page in original_pgdata['files'][file]['md5_per_page']: + if page not in restored_pgdata['files'][file]['md5_per_page']: + error_message += ( + '\n Page {0} dissappeared.\n ' + 'File: {1}\n').format( + page, + os.path.join( + restored_pgdata['pgdata'], + file + ) + ) + continue + + if original_pgdata['files'][file][ + 'md5_per_page'][page] != restored_pgdata[ + 'files'][file]['md5_per_page'][page]: + error_message += ( + '\n Page checksumm mismatch: {0}\n ' + ' PAGE Checksumm_old: {1}\n ' + ' PAGE Checksumm_new: {2}\n ' + ' File: {3}\n' + ).format( + page, + original_pgdata['files'][file][ + 'md5_per_page'][page], + restored_pgdata['files'][file][ + 'md5_per_page'][page], + os.path.join( + restored_pgdata['pgdata'], file) + ) + for page in restored_pgdata['files'][file]['md5_per_page']: + if page not in original_pgdata['files'][file]['md5_per_page']: + error_message += '\n Extra page {0}\n File: {1}\n'.format( + page, + os.path.join( + restored_pgdata['pgdata'], file)) + + else: + error_message += ( + '\nFile dissappearance.\n ' + 'File: {0}\n').format( + os.path.join(restored_pgdata['pgdata'], file) + ) + fail = True + self.assertFalse(fail, error_message) + + def get_async_connect(self, database=None, host=None, port=5432): + if not database: + database = 'postgres' + if not host: + host = '127.0.0.1' + + return psycopg2.connect( + database="postgres", + host='127.0.0.1', + port=port, + async=True + ) + + def wait(self, connection): + while True: + state = connection.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [connection.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([connection.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + + def gdb_attach(self, pid): + return GDBobj([str(pid)], self.verbose, attach=True) + + +class GdbException(Exception): + def __init__(self, message=False): + self.message = message + + def __str__(self): + return '\n ERROR: {0}\n'.format(repr(self.message)) + + +class GDBobj(ProbackupTest): + def __init__(self, cmd, verbose, attach=False): + self.verbose = verbose + + # Check gdb presense + try: + gdb_version, _ = subprocess.Popen( + ["gdb", "--version"], + stdout=subprocess.PIPE + ).communicate() + except OSError: + raise GdbException("Couldn't find gdb on the path") + + self.base_cmd = [ + 'gdb', + '--interpreter', + 'mi2', + ] + + if attach: + self.cmd = self.base_cmd + ['--pid'] + cmd + else: + self.cmd = self.base_cmd + ['--args'] + cmd + + # Get version + gdb_version_number = re.search( + b"^GNU gdb [^\d]*(\d+)\.(\d)", + gdb_version) + self.major_version = int(gdb_version_number.group(1)) + self.minor_version = int(gdb_version_number.group(2)) + + if self.verbose: + print([' '.join(map(str, self.cmd))]) + + self.proc = subprocess.Popen( + self.cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + universal_newlines=True + ) + self.gdb_pid = self.proc.pid + + # discard data from pipe, + # is there a way to do it a less derpy way? + while True: + line = self.proc.stdout.readline() + + if 'No such process' in line: + raise GdbException(line) + + if not line.startswith('(gdb)'): + pass + else: + break + + def set_breakpoint(self, location): + result = self._execute('break ' + location) + for line in result: + if line.startswith('~"Breakpoint'): + return + + elif line.startswith('^error') or line.startswith('(gdb)'): + break + + elif line.startswith('&"break'): + pass + + elif line.startswith('&"Function'): + raise GdbException(line) + + elif line.startswith('&"No line'): + raise GdbException(line) + + elif line.startswith('~"Make breakpoint pending on future shared'): + raise GdbException(line) + + raise GdbException( + 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + ) + + def run_until_break(self): + result = self._execute('run', False) + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + raise GdbException( + 'Failed to run until breakpoint.\n' + ) + + def continue_execution_until_running(self): + result = self._execute('continue') + + running = False + for line in result: + if line.startswith('*running'): + running = True + break + if line.startswith('*stopped,reason="breakpoint-hit"'): + running = False + continue + if line.startswith('*stopped,reason="exited-normally"'): + running = False + continue + return running + + def continue_execution_until_exit(self): + result = self._execute('continue', False) + + for line in result: + if line.startswith('*running'): + continue + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if ( + line.startswith('*stopped,reason="exited-normally"') or + line == '*stopped\n' + ): + return + raise GdbException( + 'Failed to continue execution until exit.\n' + ) + + def continue_execution_until_break(self, ignore_count=0): + if ignore_count > 0: + result = self._execute( + 'continue ' + str(ignore_count), + False + ) + else: + result = self._execute('continue', False) + + running = False + for line in result: + if line.startswith('*running'): + running = True + if line.startswith('*stopped,reason="breakpoint-hit"'): + return 'breakpoint-hit' + if line.startswith('*stopped,reason="exited-normally"'): + return 'exited-normally' + if running: + return 'running' + + def stopped_in_breakpoint(self): + output = [] + while True: + line = self.proc.stdout.readline() + output += [line] + if self.verbose: + print(line) + if line.startswith('*stopped,reason="breakpoint-hit"'): + return True + return False + + # use for breakpoint, run, continue + def _execute(self, cmd, running=True): + output = [] + self.proc.stdin.flush() + self.proc.stdin.write(cmd + '\n') + self.proc.stdin.flush() + + while True: + line = self.proc.stdout.readline() + output += [line] + if self.verbose: + print(repr(line)) + if line == '^done\n' or line.startswith('*stopped'): + break + if running and line.startswith('*running'): + break + return output diff --git a/tests/init_test.py b/tests/init_test.py new file mode 100644 index 00000000..0b91dafa --- /dev/null +++ b/tests/init_test.py @@ -0,0 +1,99 @@ +import os +import unittest +from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException + + +module_name = 'init' + + +class InitTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_success(self): + """Success normal init""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + self.init_pb(backup_dir) + self.assertEqual( + dir_files(backup_dir), + ['backups', 'wal'] + ) + self.add_instance(backup_dir, 'node', node) + self.assertEqual("INFO: Instance 'node' successfully deleted\n", self.del_instance(backup_dir, 'node'), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Show non-existing instance + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Delete non-existing instance + try: + self.del_instance(backup_dir, 'node1') + self.assertEqual(1, 0, 'Expecting Error due to delete of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node1' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Add instance without pgdata + try: + self.run_pb([ + "add-instance", + "--instance=node1", + "-B", backup_dir + ]) + self.assertEqual(1, 0, 'Expecting Error due to adding instance without pgdata. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_already_exist(self): + """Failure with backup catalog already existed""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + self.init_pb(backup_dir) + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_abs_path(self): + """failure with backup catalog should be given as absolute path""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + try: + self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) + self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: -B, --backup-path must be an absolute path\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/logging.py b/tests/logging.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/merge.py b/tests/merge.py new file mode 100644 index 00000000..1be3dd8b --- /dev/null +++ b/tests/merge.py @@ -0,0 +1,454 @@ +# coding: utf-8 + +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest + +module_name = "merge" + + +class MergeTest(ProbackupTest, unittest.TestCase): + + def test_merge_full_page(self): + """ + Test MERGE command, it merges FULL backup with target PAGE backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + conn.commit() + + # Do first page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do second page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + # sanity check + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Check physical correctness + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_backups(self): + """ + Test MERGE command with compressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full compressed backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do compressed page backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[1] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_merge_tablespaces(self): + """ + Some test here + """ + + def test_merge_page_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_delta_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_ptrack_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/option_test.py b/tests/option_test.py new file mode 100644 index 00000000..8bd473fa --- /dev/null +++ b/tests/option_test.py @@ -0,0 +1,218 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'option' + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_help_1(self): + """help options""" + self.maxDiff = None + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + + # @unittest.skip("skip") + def test_version_2(self): + """help options""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: + self.assertIn( + version_out.read().decode("utf-8"), + self.run_pb(["--version"]) + ) + + # @unittest.skip("skip") + def test_without_backup_path_3(self): + """backup command failure without backup mode option""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + try: + self.run_pb(["backup", "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # @unittest.skip("skip") + def test_options_4(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # backup command failure without instance option + try: + self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: required parameter not specified: --instance\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure without backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) + self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn('ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure with invalid backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) + self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid backup-mode "bad"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # delete failure without delete options + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because delete options are omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: You must specify at least one of the delete options: --expired |--wal |--backup_id\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # delete failure without ID + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node", '-i']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue("option requires an argument -- 'i'" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_options_5(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + pg_options={ + 'wal_level': 'logical', + 'max_wal_senders': '2'}) + + self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir), + self.init_pb(backup_dir)) + self.add_instance(backup_dir, 'node', node) + + node.start() + + # syntax error in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write(" = INFINITE\n") + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: syntax error in " = INFINITE"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid value in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("BACKUP_MODE=\n") + + try: + self.backup_node(backup_dir, 'node', node, backup_type=None), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid backup-mode ""\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Command line parameters should override file values + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy=1\n") + + self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') + + # User cannot send --system-identifier parameter via command line + try: + self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: option system-identifier cannot be specified in command line\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # invalid value in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("SMOOTH_CHECKPOINT=FOO\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid option in pg_probackup.conf + pbconf_path = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") + with open(pbconf_path, "a") as conf: + conf.write("TIMELINEID=1\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid option "TIMELINEID" in file "{0}"\n'.format(pbconf_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/page.py b/tests/page.py new file mode 100644 index 00000000..ef7122b6 --- /dev/null +++ b/tests/page.py @@ -0,0 +1,641 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + +module_name = 'page' + + +class PageBackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_page_vacuum_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, take second page backup, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--log-level-file=verbose']) + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_stream(self): + """ + make archive node, take full and page stream backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_archive(self): + """ + make archive node, take full and page archive backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full') + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn("INFO: Restore of backup {0} completed.".format( + full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check page and data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--log-level-file=verbose"]) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", + "--recovery-target-action=promote", + "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_delete(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + # FULL backup + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_delete_1(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + "hot_standby": "on" + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + with node.connect() as conn: + conn.execute("create table test (id int)") + for x in range(0, 8): + conn.execute( + "insert into test select i from generate_series(1,100) s(i)") + conn.commit() + self.switch_wal_segment(conn) + count1 = conn.execute("select count(*) from test") + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore it + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Check restored node + count2 = node_restored.execute("postgres", "select count(*) from test") + + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + node_restored.cleanup() + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap_1(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + node.pgbench_init(scale=10) + + # do page backup in single thread + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + self.delete_pb(backup_dir, 'node', page_id) + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.start() + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro560.py b/tests/pgpro560.py new file mode 100644 index 00000000..bf334556 --- /dev/null +++ b/tests/pgpro560.py @@ -0,0 +1,98 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'pgpro560' + + +class CheckSystemID(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro560_control_file_loss(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node with stream support, delete control file + make backup + check that backup failed + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + file = os.path.join(node.base_dir,'data', 'global', 'pg_control') + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: could not open file' in e.message + and 'pg_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_pgpro560_systemid_mismatch(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node1 and node2 + feed to backup PGDATA from node1 and PGPORT from node2 + check that backup failed + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node1.start() + node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node2.start() + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + + try: + self.backup_node(backup_dir, 'node1', node2, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node1', node2, data_dir=node1.data_dir, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589.py new file mode 100644 index 00000000..bd40f16d --- /dev/null +++ b/tests/pgpro589.py @@ -0,0 +1,80 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'pgpro589' + + +class ArchiveCheck(ProbackupTest, unittest.TestCase): + + def test_pgpro589(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check that backup status equal to ERROR + check that no files where copied to backup catalogue + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + # make erroneus archive_command + node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").rstrip().decode( + "utf-8") + + try: + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing archive wal " + "segment with start_lsn.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for WAL segment' in e.message and + 'ERROR: Switched WAL segment' in e.message and + 'could not be archived' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup should have ERROR status') + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', path) + self.assertFalse( + os.path.isfile(file), + "\n Start LSN was not found in archive but datafiles where " + "copied to backup catalogue.\n For example: {0}\n " + "It is not optimal".format(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack.py b/tests/ptrack.py new file mode 100644 index 00000000..c2d6abff --- /dev/null +++ b/tests/ptrack.py @@ -0,0 +1,1600 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from testgres import QueryException +import shutil +import sys +import time + + +module_name = 'ptrack' + + +class PtrackTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_enable(self): + """make ptrack without full backup, should result in error""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack disabled.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: Ptrack is disabled\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_disable(self): + """ + Take full backup, disable ptrack restart postgresql, + enable ptrack, restart postgresql, take ptrack backup + which should fail + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # DISABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack_enable to off") + node.restart() + + # ENABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack_enable to on") + node.restart() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack_enable was set to OFF at some" + " point after previous backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: LSN from ptrack_control', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd + ) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_uncommited_xact(self): + """make ptrack backup while there is uncommited open transaction""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + self.backup_node(backup_dir, 'node', node) + con = node.connect("postgres") + con.execute( + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + pgdata = self.pgdata_content(node.data_dir) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_full(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + # create async connection + conn = self.get_async_connect(port=node.port) + + self.wait(conn) + + acurs = conn.cursor() + acurs.execute("select pg_backend_pid()") + + self.wait(conn) + pid = acurs.fetchall()[0][0] + print(pid) + + gdb = self.gdb_attach(pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + if not gdb.continue_execution_until_running(): + print('Failed gdb continue') + exit(1) + + acurs.execute("VACUUM FULL t_heap") + + if gdb.stopped_in_breakpoint(): + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_truncate(self): + """make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take ptrack backup, take second ptrack backup, + restore last ptrack backup and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, + ignore_ptrack=False + ) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_simple(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Logical comparison + self.assertEqual( + result, + node_restored.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_get_block(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('make_pagemap_from_ptrack') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_stream(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql("postgres", "create sequence t_seq") + node.safe_psql( + "postgres", + "create table t_heap as select i as id, nextval('t_seq')" + " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" + " as tsvector from generate_series(0,100) i" + ) + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, nextval('t_seq') as t_seq," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', + node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=["-j", "4", "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=["-j", "4", "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_archive(self): + """make archive node, make full and ptrack backups, + check data correctness in restored instance""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as" + " select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node(backup_dir, 'node', node) + full_target_time = self.show_pb( + backup_dir, 'node', full_backup_id)['recovery-time'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack') + ptrack_target_time = self.show_pb( + backup_dir, 'node', ptrack_backup_id)['recovery-time'] + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--recovery-target-action=promote", + "--time={0}".format(full_target_time)] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=[ + "-j", "4", + "--time={0}".format(ptrack_target_time), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_pgpro417(self): + """Make node, take full backup, take ptrack backup, + delete ptrack backup. Try to take ptrack backup, + which should fail""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': + 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql( + "postgres", + "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["--stream"]) + + start_lsn_full = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + start_lsn_ptrack = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + self.delete_pb(backup_dir, 'node', backup_id) + + # SECOND PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_pgpro417(self): + """ + Make archive node, take full backup, take page backup, + delete page backup. Try to take ptrack backup, which should fail + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.delete_pb(backup_dir, 'node', backup_id) +# sys.exit(1) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0}\n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_full_pgpro417(self): + """ + Make node, take two full backups, delete full second backup. + Try to take ptrack backup, which should fail + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector " + " from generate_series(0,100) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # SECOND FULL BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + + self.delete_pb(backup_dir, 'node', backup_id) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertTrue( + "ERROR: LSN from ptrack_control" in e.message and + "Create new full backup before " + "an incremental one" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take ptrack backup, + restore database and check it presense + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream", "--log-level-file=verbose"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND PTRACK BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_ptrack(self): + """Make node, create tablespace with table, take full backup, + alter tablespace location, take ptrack backup, restore database.""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new" + ) + + # sys.exit(1) + # PTRACK BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + # node.stop() + # node.cleanup() + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ), + "--recovery-target-action=promote" + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_ptrack(self): + """Make node, create tablespace with database," + " take full backup, alter tablespace location," + " take ptrack backup, restore database.""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE TABLESPACE + self.create_tblspace_in_node(node, 'somedata') + + # ALTER DATABASE + node.safe_psql( + "template1", + "alter database postgres set tablespace somedata") + + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", '--log-level-file=verbose']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + node.stop() + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', + node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata'))]) + + # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_drop_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to tablespace 'somedata' + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # Move table back to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # DROP TABLESPACE 'somedata' + node.safe_psql( + "postgres", "drop tablespace somedata") + # THIRD PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + tblspace = self.get_tblspace_path(node, 'somedata') + node.cleanup() + shutil.rmtree(tblspace, ignore_errors=True) + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + node.start() + + tblspc_exist = node.safe_psql( + "postgres", + "select exists(select 1 from " + "pg_tablespace where spcname = 'somedata')") + + if tblspc_exist.rstrip() == 't': + self.assertEqual( + 1, 0, + "Expecting Error because " + "tablespace 'somedata' should not be present") + + result_new = node.safe_psql("postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_alter_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + tblspc_path = self.get_tblspace_path(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to separate tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from t_heap") + + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"]) + + # GET PHYSICAL CONTENT FROM NODE + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore ptrack backup + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + # COMPARE LOGICAL CONTENT + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + restored_node.cleanup() + shutil.rmtree(tblspc_path_new, ignore_errors=True) + + # Move table to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore second ptrack backup and check table consistency + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_multiple_segments(self): + """ + Make node, create table, alter table tablespace, + take ptrack backup, move table from tablespace, take ptrack backup + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'ptrack_enable': 'on', 'fsync': 'off', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PTRACK STUFF + idx_ptrack = {'type': 'heap'} + idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') + idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') + idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], idx_ptrack['old_size']) + + pgbench = node.pgbench(options=['-T', '150', '-c', '2', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + idx_ptrack['new_size'] = self.get_fork_size( + node, + 'pgbench_accounts' + ) + idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], + idx_ptrack['new_size'] + ) + idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, + idx_ptrack['path'] + ) + self.check_ptrack_sanity(idx_ptrack) + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--log-level-file=verbose"] + ) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, + 'somedata_restored' + ) + + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", + "select * from pgbench_accounts" + ) + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_atexit_fail(self): + """ + Take backups of every available types and check that PTRACK is clean + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'max_connections': '15'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=[ + "--stream", "-j 30", + "--log-level-file=verbose"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are opening too many connections" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'setting its status to ERROR', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + self.assertEqual( + node.safe_psql( + "postgres", + "select * from pg_is_in_backup()").rstrip(), + "f") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py new file mode 100644 index 00000000..f4350af0 --- /dev/null +++ b/tests/ptrack_clean.py @@ -0,0 +1,253 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import time + + +module_name = 'ptrack_clean' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean(self): + """Take backups of every available types and check that PTRACK is clean""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata " + "as select i as id, nextval('t_seq') as t_seq, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + options=['-j10', '--stream']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['-j10', '--log-level-file=verbose']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['-j10']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean_replica(self): + """Take backups of every available types from master and check that PTRACK on replica is clean""" + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='ptrack', + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='page', + options=[ + '-j10', '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py new file mode 100644 index 00000000..784751ef --- /dev/null +++ b/tests/ptrack_cluster.py @@ -0,0 +1,268 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from time import sleep +from sys import exit + + +module_name = 'ptrack_cluster' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_cluster_on_btree(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_btree') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_gist(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_gist') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_btree_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_btree') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + #@unittest.skip("skip") + def test_ptrack_cluster_on_gist_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_gist') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py new file mode 100644 index 00000000..98c20914 --- /dev/null +++ b/tests/ptrack_move_to_tablespace.py @@ -0,0 +1,57 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_move_to_tablespace' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql("postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # Move table and indexes and make checkpoint + for i in idx_ptrack: + if idx_ptrack[i]['type'] == 'heap': + node.safe_psql('postgres', 'alter table {0} set tablespace somedata;'.format(i)) + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql('postgres', 'alter index {0} set tablespace somedata'.format(i)) + node.safe_psql('postgres', 'checkpoint') + + # Check ptrack files + for i in idx_ptrack: + if idx_ptrack[i]['type'] == 'seq': + continue + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py new file mode 100644 index 00000000..8569ef59 --- /dev/null +++ b/tests/ptrack_recovery.py @@ -0,0 +1,58 @@ +import os +import unittest +from sys import exit +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_recovery' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table + node.safe_psql("postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = int(self.get_fork_size(node, i)) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + if not node.status(): + node.start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + for i in idx_ptrack: + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py new file mode 100644 index 00000000..928608c4 --- /dev/null +++ b/tests/ptrack_truncate.py @@ -0,0 +1,130 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_truncate' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'truncate t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + replica.safe_psql('postgres', 'truncate t_heap') + replica.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py new file mode 100644 index 00000000..0409cae3 --- /dev/null +++ b/tests/ptrack_vacuum.py @@ -0,0 +1,152 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make FULL backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py new file mode 100644 index 00000000..f0cd3bbd --- /dev/null +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -0,0 +1,136 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_bits_frozen' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_frozen(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum freeze t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_bits_frozen_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take PTRACK backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'vacuum freeze t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py new file mode 100644 index 00000000..45a8d9b6 --- /dev/null +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -0,0 +1,67 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_bits_visibility' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_visibility(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py new file mode 100644 index 00000000..ec12c9e2 --- /dev/null +++ b/tests/ptrack_vacuum_full.py @@ -0,0 +1,140 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_full' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum full t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take FULL backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum full t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py new file mode 100644 index 00000000..5c84c7e8 --- /dev/null +++ b/tests/ptrack_vacuum_truncate.py @@ -0,0 +1,142 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_truncate' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id > 128;') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take PTRACK backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id > 128;') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/replica.py b/tests/replica.py new file mode 100644 index 00000000..d74c375c --- /dev/null +++ b/tests/replica.py @@ -0,0 +1,293 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess +from sys import exit +import time + + +module_name = 'replica' + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_ptrack_backup(self): + """ + make node, take full backup, restore it and make replica from it, + take full stream backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} + ) + master.start() + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + # CREATE TABLE + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # take full backup and restore it + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica) + + # Check data correctness on replica + replica.slow_start(replica=True) + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take PTRACK backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PTRACK BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_archive_page_backup(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data + # equal to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=300', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + options=[ + '--archive-timeout=300', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_make_replica_via_restore(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node( + backup_dir, 'master', replica, + options=['-R', '--recovery-target-action=promote']) + + # Settings for Replica + # self.set_replica(master, replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/restore_test.py b/tests/restore_test.py new file mode 100644 index 00000000..c33a1e29 --- /dev/null +++ b/tests/restore_test.py @@ -0,0 +1,1243 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from datetime import datetime +import sys +import time + + +module_name = 'restore' + + +class RestoreTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_restore_full_to_latest(self): + """recovery to latest from full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # 1 - Test recovery from latest + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # 2 - Test that recovery.conf was created + recovery_conf = os.path.join(node.data_dir, "recovery.conf") + self.assertEqual(os.path.isfile(recovery_conf), True) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_page_to_latest(self): + """recovery to latest from full + page backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_specific_timeline(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_tli = int( + node.get_control_data()["Latest checkpoint's TimeLineID"]) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # Correct Backup must be choosen for restore + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--timeline={0}".format(target_tli), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + recovery_target_timeline = self.get_recovery_conf( + node)["recovery_target_timeline"] + self.assertEqual(int(recovery_target_timeline), target_tli) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_time(self): + """recovery to target time""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.append_conf("postgresql.auto.conf", "TimeZone = Europe/Moscow") + node.start() + + node.pgbench_init(scale=2) + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(target_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_xid_inclusive(self): + """recovery to target xid""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--xid={0}'.format(target_xid), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_xid_not_inclusive(self): + """recovery with target inclusive false""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = result[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + '--xid={0}'.format(target_xid), + "--inclusive=false", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_not_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "--inclusive=false", + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_archive(self): + """recovery to latest from archive full+ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_ptrack(self): + """recovery to latest from archive full+ptrack+ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_stream(self): + """recovery in stream mode to latest from full + ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_under_load(self): + """ + recovery to latest from full + ptrack backups + with loads when ptrack backup do + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + pgbench.wait() + pgbench.stdout.close() + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_under_load_ptrack(self): + """ + recovery to latest from full + page backups + with loads when full backup do + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # wal_segment_size = self.guc_wal_segment_size(node) + node.pgbench_init(scale=2) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + self.backup_node(backup_dir, 'node', node) + + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + + node.stop() + node.cleanup() + # self.wrong_wal_clean(node, wal_segment_size) + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_1(self): + """recovery using tablespace-mapping option""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") + con.execute("INSERT INTO test VALUES (1)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # 1 - Try to restore to existing directory + node.stop() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore destionation is not empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: restore destination is not empty: "{0}"\n'.format( + node.data_dir), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 2 - Try to restore to existing tablespace directory + node.cleanup() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore tablespace destination is " + "not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: restore tablespace destination ' + 'is not empty: "{0}"\n'.format(tblspc_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 3 - Restore using tablespace-mapping + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(result[0][0], 1) + + # 4 - Restore using tablespace-mapping using page backup + self.backup_node(backup_dir, 'node', node) + with node.connect("postgres") as con: + con.execute("INSERT INTO test VALUES (2)") + con.commit() + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + show_pb = self.show_pb(backup_dir, 'node') + self.assertEqual(show_pb[1]['status'], "OK") + self.assertEqual(show_pb[2]['status'], "OK") + + node.stop() + node.cleanup() + tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page), + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(result[0][0], 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_2(self): + """recovery using tablespace-mapping option and page backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Full backup + self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # First page backup + self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") + + # Create tablespace table + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CHECKPOINT") + con.connection.autocommit = False + con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") + con.execute( + "INSERT INTO tbl1 SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Second page backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") + + node.stop() + node.cleanup() + + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + count = node.execute("postgres", "SELECT count(*) FROM tbl1") + self.assertEqual(count[0][0], 4) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.safe_psql("postgres", "select pg_switch_xlog()") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_pitr(self): + """ + make node with archiving, make stream backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_archive_pitr_2(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_restore_to_restore_point(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,10000)") + result = node.safe_psql( + "postgres", + "select * from t_heap") + node.safe_psql( + "postgres", "select pg_create_restore_point('savepoint')") + node.safe_psql( + "postgres", + "create table t_heap_1 as select generate_series(0,10000)") + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + "--recovery-target-name=savepoint", + "--recovery-target-action=promote"]) + + node.slow_start() + + result_new = node.safe_psql("postgres", "select * from t_heap") + res = node.psql("postgres", "select * from t_heap_1") + self.assertEqual( + res[0], 1, + "Table t_heap_1 should not exist in restored instance") + + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/retention_test.py b/tests/retention_test.py new file mode 100644 index 00000000..652f7c39 --- /dev/null +++ b/tests/retention_test.py @@ -0,0 +1,178 @@ +import os +import unittest +from datetime import datetime, timedelta +from .helpers.ptrack_helpers import ProbackupTest + + +module_name = 'retention' + + +class RetentionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_retention_redundancy_1(self): + """purge backups using redundancy-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backups to be keeped + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + log = self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Check that WAL segments were deleted + min_wal = None + max_wal = None + for line in log.splitlines(): + if line.startswith("INFO: removed min WAL segment"): + min_wal = line[31:-1] + elif line.startswith("INFO: removed max WAL segment"): + max_wal = line[31:-1] + + if not min_wal: + self.assertTrue(False, "min_wal is empty") + + if not max_wal: + self.assertTrue(False, "max_wal is not set") + + for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): + if not wal_name.endswith(".backup"): + # wal_name_b = wal_name.encode('ascii') + self.assertEqual(wal_name[8:] > min_wal[8:], True) + self.assertEqual(wal_name[8:] > max_wal[8:], True) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# @unittest.skip("123") + def test_retention_window_2(self): + """purge backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with open( + os.path.join( + backup_dir, + 'backups', + 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + conf.write("retention-window = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# @unittest.skip("123") + def test_retention_wal(self): + """purge backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + # Take FULL BACKUP + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--retention-window=2']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/show_test.py b/tests/show_test.py new file mode 100644 index 00000000..931da184 --- /dev/null +++ b/tests/show_test.py @@ -0,0 +1,203 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'show' + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_1(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_json(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_corrupt_2(self): + """Status CORRUPT""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete file which belong to backup + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node', backup_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup corrupted.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'data files are corrupted\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_no_control_file(self): + """backup.control doesn't exist""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + os.remove(file) + + self.assertIn('control file "{0}" doesn\'t exist'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_empty_control_file(self): + """backup.control is empty""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # truncate backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'w') + fd.close() + + self.assertIn('control file "{0}" is empty'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_control_file(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # corrupt backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'a') + fd.write("statuss = OK") + fd.close() + + self.assertIn('invalid option "statuss" in file'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py new file mode 100644 index 00000000..ab091c57 --- /dev/null +++ b/tests/validate_test.py @@ -0,0 +1,1730 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from sys import exit +import time + + +module_name = 'validate' + + +class ValidateTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_validate_wal_unreal_values(self): + """ + make node with archiving, make archive backup + validate to both real and unreal values + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + + pgbench.wait() + pgbench.stdout.close() + + target_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + after_backup_time = datetime.now().replace(second=0, microsecond=0) + + # Validate to real time + self.assertIn( + "INFO: backup validation completed successfully", + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(target_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Validate to unreal time + unreal_time_1 = after_backup_time - timedelta(days=2) + try: + self.validate_pb( + backup_dir, 'node', options=["--time={0}".format( + unreal_time_1)]) + self.assertEqual( + 1, 0, + "Expecting Error because of validation to unreal time.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Full backup satisfying target options is not found.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Validate to unreal time #2 + unreal_time_2 = after_backup_time + timedelta(days=2) + try: + self.validate_pb(backup_dir, 'node', options=["--time={0}".format(unreal_time_2)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('ERROR: not enough WAL records to time' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Validate to real xid + target_xid = None + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + self.switch_wal_segment(node) + + self.assertIn("INFO: backup validation completed successfully", + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Validate to unreal xid + unreal_xid = int(target_xid) + 1000 + try: + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('ERROR: not enough WAL records to xid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Validate with backup ID + self.assertIn("INFO: Validating backup {0}".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} data files are valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} WAL segments are valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} is valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Validate of backup {0} completed".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backup(self): + """make archive node, take FULL, PAGE1, PAGE2 backups, corrupt file in PAGE1 backup, + run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") + # PAGE2 + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file + file = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path) + with open(file, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id_2, + options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format(backup_id_2) in e.message + and 'ERROR: Backup {0} is corrupt'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups(self): + """make archive node, take FULL, PAGE1, PAGE2 backups, + corrupt file in FULL and PAGE1 backupd, run validate on PAGE1, + expect FULL and PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap_1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Corrupt some file in PAGE1 backup + file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path_t_heap_1) + with open(file_page1, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE1 + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id_2, + options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('INFO: Validating parents for backup {0}'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent'.format(backup_id_2) in e.message + and 'WARNING: Backup {0} is orphaned because his parent'.format(backup_id_3) in e.message + and 'ERROR: Backup {0} is orphan.'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups_1(self): + """make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 backups, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + backup_id_4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE4 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + backup_id_6 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_7 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join( + backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join( + backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb( + backup_dir, 'node', + backup_id=backup_id_4, + options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_page1) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_specific_target_corrupted_intermediate_backups(self): + """make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 backups, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3 to specific xid, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") + backup_id_4 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE4 + target_xid = node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i RETURNING (xmin)")[0][0] + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + backup_id_6 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_7 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join(backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb(backup_dir, 'node', + options=['--log-level-file=verbose', '-i', backup_id_4, '--xid={0}'.format(target_xid)]) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and 'INFO: Backup {0} data files are valid'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_2) in e.message + and 'INFO: Backup {0} data files are valid'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_3) in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_page1) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_page(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in PAGE1 backup and run validate on instance, + expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + # FULL1 + backup_id_4 = self.backup_node( + backup_dir, 'node', node) + # PAGE3 + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join( + backup_dir, 'backups/node', backup_id_2, + 'database', file_path_t_heap1) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb( + backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message, + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_5) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_3) in e.message and + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_3, backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_full) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Some backups are not valid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full_and_try_restore(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, + try to restore backup with --no-validation option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + node.cleanup() + restore_out = self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id_5), + restore_out, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_1(self): + """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(42) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_2(self): + """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + target_xid = None + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(128) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_1(self): + """make archive node, make archive full backup, + delete from archive wal segment which belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'CORRUPT' + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + backup_id = self.backup_node(backup_dir, 'node', node) + + # Delete wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + file = os.path.join(backup_dir, 'wal', 'node', wals[-1]) + os.remove(file) + + # cut out '.gz' + if self.archive_compress: + file = file[:-3] + + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: WAL segment \"{0}\" is absent".format( + file) in e.message and + "WARNING: There are not enough WAL records to consistenly " + "restore backup {0}".format(backup_id) in e.message and + "WARNING: Backup {0} WAL segments are corrupted".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup {0} should have STATUS "CORRUPT"') + + # Run validate again + try: + self.validate_pb(backup_dir, 'node', backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'INFO: Revalidating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Backup {0} is corrupt.'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_between_backups(self): + """ + make archive node, make full backup, corrupt all wal files, + run validate to real xid, expect errors + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + if self.get_version(node) < self.version_to_num('10.0'): + walfile = node.safe_psql( + 'postgres', + 'select pg_xlogfile_name(pg_current_xlog_location())').rstrip() + else: + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() + + if self.archive_compress: + walfile = walfile + '.gz' + self.switch_wal_segment(node) + + # generate some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: + f.seek(9000) + f.write(b"b") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: not enough WAL records to xid' in e.message and + 'WARNING: recovery can be done up to time' in e.message and + "ERROR: not enough WAL records to xid {0}\n".format( + target_xid), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_2(self): + """ + make node with archiving + make archive backup + delete from archive wal segment which DO NOT belong to this backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'ERROR' + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( + wals_dir, f)) and not f.endswith('.backup')] + wals = map(str, wals) + file = os.path.join(wals_dir, max(wals)) + os.remove(file) + if self.archive_compress: + file = file[:-3] + + # Try to restore + try: + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for LSN' in e.message and + 'in archived WAL segment' in e.message and + 'WARNING: could not read WAL record at' in e.message and + 'ERROR: WAL segment "{0}" is absent\n'.format( + file) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro702_688(self): + """make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + recovery_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + try: + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(recovery_time)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WAL archive is empty. You cannot restore backup to a ' + 'recovery target without WAL archive', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro688(self): + """make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + + self.validate_pb(backup_dir, 'node', options=["--time={0}".format(recovery_time)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro561(self): + """ + make node with archiving, make stream backup, + restore it to node1, check that archiving is not successful on node1 + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node( + base_dir="{0}/{1}/node1".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + self.set_archiving(backup_dir, 'node1', node1) + node1.start() + + backup_id = self.backup_node( + backup_dir, 'node1', node1, options=["--stream"]) + + node2 = self.make_simple_node( + base_dir="{0}/{1}/node2".format(module_name, fname)) + node2.cleanup() + + node1.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node( + backup_dir, 'node1', node1, + backup_type='page', options=["--stream"]) + self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + node2.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) + node2.slow_start() + + timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] + timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual( + timeline_node1, timeline_node2, + "Timelines on Master and Node1 should be equal. " + "This is unexpected") + + archive_command_node1 = node1.safe_psql( + "postgres", "show archive_command") + archive_command_node2 = node2.safe_psql( + "postgres", "show archive_command") + self.assertEqual( + archive_command_node1, archive_command_node2, + "Archive command on Master and Node should be equal. " + "This is unexpected") + + # result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + ## self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) + # if result == "": + # self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') + + self.switch_wal_segment(node1) + self.switch_wal_segment(node2) + time.sleep(5) + + log_file = os.path.join(node2.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse( + 'pg_probackup archive-push completed successfully' in log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_full(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and three page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption and run valudate again, check that + second full backup and his page backups are OK + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "alter system set archive_command = 'false'") + node.reload() + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=1s']) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + pass + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.set_archiving(backup_dir, 'node', node) + node.reload() + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue( + self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue( + self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + + os.rename(file_new, file) + try: + self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_full_1(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and four page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption from full backup and corrupt his second page backup + run valudate again, check that + second full backup and his firts page backups are OK, + second page should be CORRUPT + third page should be ORPHAN + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_page = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + os.rename(file_new, file) + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id_page, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_file_size_corruption_no_validate(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + # initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4"], async=False, gdb=False) + + node.stop() + node.cleanup() + + # Let`s do file corruption + with open(os.path.join(backup_dir, "backups", 'node', backup_id, "database", heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + except ProbackupException as e: + self.assertTrue("ERROR: Data files restoring failed" in e.message, repr(e.message)) + print "\nExpected error: \n" + e.message + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/travis/backup_restore.sh b/travis/backup_restore.sh new file mode 100644 index 00000000..7fe1cfd8 --- /dev/null +++ b/travis/backup_restore.sh @@ -0,0 +1,66 @@ +#!/bin/sh -ex + +# vars +export PGVERSION=9.5.4 +export PATH=$PATH:/usr/pgsql-9.5/bin +export PGUSER=pgbench +export PGDATABASE=pgbench +export PGDATA=/var/lib/pgsql/9.5/data +export BACKUP_PATH=/backups +export ARCLOG_PATH=$BACKUP_PATH/backup/pg_xlog +export PGDATA2=/var/lib/pgsql/9.5/data2 +export PGBENCH_SCALE=100 +export PGBENCH_TIME=60 + +# prepare directory +cp -a /tests /build +pushd /build + +# download postgresql +yum install -y wget +wget -k https://ftp.postgresql.org/pub/source/v$PGVERSION/postgresql-$PGVERSION.tar.gz -O postgresql.tar.gz +tar xf postgresql.tar.gz + +# install pg_probackup +yum install -y https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-7-x86_64/pgdg-centos95-9.5-2.noarch.rpm +yum install -y postgresql95-devel make gcc readline-devel openssl-devel pam-devel libxml2-devel libxslt-devel +make top_srcdir=postgresql-$PGVERSION +make install top_srcdir=postgresql-$PGVERSION + +# initalize cluster and database +yum install -y postgresql95-server +su postgres -c "/usr/pgsql-9.5/bin/initdb -D $PGDATA -k" +cat < $PGDATA/pg_hba.conf +local all all trust +host all all 127.0.0.1/32 trust +local replication pgbench trust +host replication pgbench 127.0.0.1/32 trust +EOF +cat < $PGDATA/postgresql.auto.conf +max_wal_senders = 2 +wal_level = logical +wal_log_hints = on +EOF +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA" +su postgres -c "createdb -U postgres $PGUSER" +su postgres -c "createuser -U postgres -a -d -E $PGUSER" +pgbench -i -s $PGBENCH_SCALE + +# Count current +COUNT=$(psql -Atc "select count(*) from pgbench_accounts") +pgbench -s $PGBENCH_SCALE -T $PGBENCH_TIME -j 2 -c 10 & + +# create backup +pg_probackup init +pg_probackup backup -b full --disable-ptrack-clear --stream -v +pg_probackup show +sleep $PGBENCH_TIME + +# restore from backup +chown -R postgres:postgres $BACKUP_PATH +su postgres -c "pg_probackup restore -D $PGDATA2" + +# start backup server +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl stop -w -D $PGDATA" +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA2" +( psql -Atc "select count(*) from pgbench_accounts" | grep $COUNT ) || (cat $PGDATA2/pg_log/*.log ; exit 1) diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 00000000..14864181 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 00000000..c869e485 --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 00000000..a4f75553 --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + + From d29aa8b0b4ce8a5e3bd3c7b5dfbbf040d7029db5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 31 Oct 2018 09:47:53 +0300 Subject: [PATCH 07/37] PGPRO-2095: backup from replica without connection to master for PostgreSQL >= 9.6 --- src/backup.c | 81 +++++++++++++++++++++++++++++++++---------- src/pg_probackup.h | 1 + src/util.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 18 deletions(-) diff --git a/src/backup.c b/src/backup.c index 0dd681fa..602ab823 100644 --- a/src/backup.c +++ b/src/backup.c @@ -475,6 +475,8 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; + pgFile *pg_control = NULL; + elog(LOG, "Database backup start"); /* Initialize size summary */ @@ -754,9 +756,37 @@ do_backup_instance(void) parray_free(prev_backup_filelist); } + /* Copy pg_control in case of backup from replica >= 9.6 */ + if (current.from_replica && !exclusive_backup) + { + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); + + if (strcmp(tmp_file->name, "pg_control") == 0) + { + pg_control = tmp_file; + break; + } + } + + if (!pg_control) + elog(ERROR, "Failed to locate pg_control in copied files"); + + if (is_remote_backup) + remote_copy_file(NULL, pg_control); + else + if (!copy_file(pgdata, database_path, pg_control)) + elog(ERROR, "Failed to copy pg_control"); + } + + /* Notify end of backup */ pg_stop_backup(¤t); + if (current.from_replica && !exclusive_backup) + set_min_recovery_point(pg_control, database_path, current.stop_lsn); + /* Add archived xlog files into the list of files of this backup */ if (stream_wal) { @@ -883,7 +913,7 @@ do_backup(time_t start_time) } } - if (current.from_replica) + if (current.from_replica && exclusive_backup) { /* Check master connection options */ if (master_host == NULL) @@ -1089,8 +1119,11 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) params[0] = label; - /* For replica we call pg_start_backup() on master */ - conn = (backup->from_replica) ? master_conn : backup_conn; + /* For 9.5 replica we call pg_start_backup() on master */ + if (backup->from_replica && exclusive_backup) + conn = master_conn; + else + conn = backup_conn; /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; @@ -1118,16 +1151,21 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) PQclear(res); - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE && + (!(backup->from_replica && !exclusive_backup))) /* * Switch to a new WAL segment. It is necessary to get archived WAL * segment, which includes start LSN of current backup. + * Don`t do this for replica backups unless it`s PG 9.5 */ pg_switch_wal(conn); + //elog(INFO, "START LSN: %X/%X", + // (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn)); + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) /* In PAGE mode wait for current segment... */ - wait_wal_lsn(backup->start_lsn, true, false); + wait_wal_lsn(backup->start_lsn, true, false); /* * Do not wait start_lsn for stream backup. * Because WAL streaming will start after pg_start_backup() in stream @@ -1669,7 +1707,7 @@ pg_stop_backup(pgBackup *backup) PGresult *tablespace_map_content = NULL; uint32 lsn_hi; uint32 lsn_lo; - XLogRecPtr restore_lsn = InvalidXLogRecPtr; + //XLogRecPtr restore_lsn = InvalidXLogRecPtr; int pg_stop_backup_timeout = 0; char path[MAXPGPATH]; char backup_label[MAXPGPATH]; @@ -1689,16 +1727,21 @@ pg_stop_backup(pgBackup *backup) if (!backup_in_progress) elog(ERROR, "backup is not in progress"); - /* For replica we call pg_stop_backup() on master */ - conn = (current.from_replica) ? master_conn : backup_conn; + /* For 9.5 replica we call pg_stop_backup() on master */ + if (current.from_replica && exclusive_backup) + conn = master_conn; + else + conn = backup_conn; /* Remove annoying NOTICE messages generated by backend */ res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); PQclear(res); - /* Create restore point */ - if (backup != NULL) + /* Create restore point + * only if it`s backup from master, or exclusive replica(wich connects to master) + */ + if (backup != NULL && (!current.from_replica || (current.from_replica && exclusive_backup))) { const char *params[1]; char name[1024]; @@ -1716,7 +1759,7 @@ pg_stop_backup(pgBackup *backup) /* Extract timeline and LSN from the result */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); /* Calculate LSN */ - restore_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; + //restore_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; PQclear(res); } @@ -1830,10 +1873,10 @@ pg_stop_backup(pgBackup *backup) /* Calculate LSN */ stop_backup_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - if (!XRecOffIsValid(stop_backup_lsn)) - { - stop_backup_lsn = restore_lsn; - } + //if (!XRecOffIsValid(stop_backup_lsn)) + //{ + // stop_backup_lsn = restore_lsn; + //} if (!XRecOffIsValid(stop_backup_lsn)) elog(ERROR, "Invalid stop_backup_lsn value %X/%X", @@ -1939,7 +1982,7 @@ pg_stop_backup(pgBackup *backup) stream_xlog_path[MAXPGPATH]; /* Wait for stop_lsn to be received by replica */ - if (backup->from_replica) + if (current.from_replica) wait_replica_wal_lsn(stop_backup_lsn, false); /* * Wait for stop_lsn to be archived or streamed. @@ -1962,10 +2005,12 @@ pg_stop_backup(pgBackup *backup) elog(LOG, "Getting the Recovery Time from WAL"); + /* iterate over WAL from stop_backup lsn to start_backup lsn */ if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, backup->start_lsn, backup->stop_lsn, &backup->recovery_time, &backup->recovery_xid)) { + elog(LOG, "Failed to find Recovery Time in WAL. Forced to trust current_timestamp"); backup->recovery_time = recovery_time; backup->recovery_xid = recovery_xid; } @@ -2074,7 +2119,7 @@ backup_files(void *arg) elog(ERROR, "interrupted during backup"); if (progress) - elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, n_backup_files_list, file->path); /* stat file to check its current state */ @@ -2168,7 +2213,7 @@ backup_files(void *arg) file->path, file->write_size); } else - elog(LOG, "unexpected file type %d", buf.st_mode); + elog(WARNING, "unexpected file type %d", buf.st_mode); } /* Close connection */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e337771e..b75bb581 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -555,6 +555,7 @@ extern uint64 get_system_identifier(char *pgdata); extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern uint32 get_xlog_seg_size(char *pgdata_path); +extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); extern void sanityChecks(void); extern void time2iso(char *buf, size_t len, time_t time); diff --git a/src/util.c b/src/util.c index 4eefa788..5f059c37 100644 --- a/src/util.c +++ b/src/util.c @@ -14,6 +14,8 @@ #include +#include + const char * base36enc(long unsigned int value) { @@ -100,6 +102,44 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) checkControlFile(ControlFile); } +/* + * Write ControlFile to pg_control + */ +static void +writeControlFile(ControlFileData *ControlFile, char *path) +{ + int fd; + char *buffer = NULL; + +#if PG_VERSION_NUM >= 100000 + int ControlFileSize = PG_CONTROL_FILE_SIZE; +#else + int ControlFileSize = PG_CONTROL_SIZE; +#endif + + /* copy controlFileSize */ + buffer = pg_malloc(ControlFileSize); + memcpy(buffer, &ControlFile, sizeof(ControlFileData)); + + /* Write pg_control */ + unlink(path); + fd = open(path, + O_RDWR | O_CREAT | O_EXCL | PG_BINARY, + S_IRUSR | S_IWUSR); + + if (fd < 0) + elog(ERROR, "Failed to open file: %s", path); + + if (write(fd, buffer, ControlFileSize) != ControlFileSize) + elog(ERROR, "Failed to overwrite file: %s", path); + + if (fsync(fd) != 0) + elog(ERROR, "Failed to fsync file: %s", path); + + pg_free(buffer); + close(fd); +} + /* * Utility shared by backup and restore to fetch the current timeline * used by a node. @@ -250,6 +290,52 @@ get_data_checksum_version(bool safe) return ControlFile.data_checksum_version; } +/* MinRecoveryPoint 'as-is' is not to be trusted + * Use STOP LSN instead + */ +void +set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + char fullpath[MAXPGPATH]; + + elog(LOG, "Setting minRecPoint to STOP LSN: %X/%X", + (uint32) (stop_backup_lsn >> 32), + (uint32) stop_backup_lsn); + + /* Path to pg_control in backup */ + snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); + + /* First fetch file... */ + buffer = slurpFile(backup_path, XLOG_CONTROL_FILE, &size, false); + if (buffer == NULL) + elog(ERROR, "ERROR"); + + digestControlFile(&ControlFile, buffer, size); + + ControlFile.minRecoveryPoint = stop_backup_lsn; + + /* Update checksum in pg_control header */ + INIT_CRC32C(ControlFile.crc); + COMP_CRC32C(ControlFile.crc, + (char *) &ControlFile, + offsetof(ControlFileData, crc)); + FIN_CRC32C(ControlFile.crc); + + /* paranoia */ + checkControlFile(&ControlFile); + + /* update pg_control */ + writeControlFile(&ControlFile, fullpath); + + /* Update pg_control checksum in backup_list */ + file->crc = pgFileGetCRC(fullpath, false); + + pg_free(buffer); +} + /* * Convert time_t value to ISO-8601 format string. Always set timezone offset. From 96b4aa791d0797f973703965a9805fcd977257f5 Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Wed, 31 Oct 2018 13:44:16 +0300 Subject: [PATCH 08/37] ICU: fix: run default collation tests during make (install)check-world Thanks to Alexander Lakhin for reporting this. --- .gitignore | 45 + .travis.yml | 7 + COPYRIGHT | 29 + Makefile | 87 + README.md | 100 + doit.cmd | 1 + doit96.cmd | 1 + gen_probackup_project.pl | 190 ++ msvs/pg_probackup.sln | 28 + msvs/template.pg_probackup.vcxproj | 212 ++ msvs/template.pg_probackup96.vcxproj | 210 ++ msvs/template.pg_probackup_2.vcxproj | 203 ++ src/archive.c | 113 + src/backup.c | 2701 ++++++++++++++++++++++++ src/catalog.c | 915 ++++++++ src/configure.c | 490 +++++ src/data.c | 1407 ++++++++++++ src/delete.c | 464 ++++ src/dir.c | 1491 +++++++++++++ src/fetch.c | 116 + src/help.c | 605 ++++++ src/init.c | 108 + src/merge.c | 526 +++++ src/parsexlog.c | 1039 +++++++++ src/pg_probackup.c | 634 ++++++ src/pg_probackup.h | 620 ++++++ src/restore.c | 920 ++++++++ src/show.c | 500 +++++ src/status.c | 118 ++ src/util.c | 349 +++ src/utils/json.c | 134 ++ src/utils/json.h | 33 + src/utils/logger.c | 621 ++++++ src/utils/logger.h | 54 + src/utils/parray.c | 196 ++ src/utils/parray.h | 35 + src/utils/pgut.c | 2417 +++++++++++++++++++++ src/utils/pgut.h | 238 +++ src/utils/thread.c | 102 + src/utils/thread.h | 35 + src/validate.c | 354 ++++ tests/Readme.md | 24 + tests/__init__.py | 69 + tests/archive.py | 833 ++++++++ tests/auth_test.py | 391 ++++ tests/backup_test.py | 522 +++++ tests/cfs_backup.py | 1161 ++++++++++ tests/cfs_restore.py | 450 ++++ tests/cfs_validate_backup.py | 25 + tests/compression.py | 496 +++++ tests/delete_test.py | 203 ++ tests/delta.py | 1265 +++++++++++ tests/exclude.py | 164 ++ tests/expected/option_help.out | 95 + tests/expected/option_version.out | 1 + tests/false_positive.py | 333 +++ tests/helpers/__init__.py | 2 + tests/helpers/cfs_helpers.py | 91 + tests/helpers/ptrack_helpers.py | 1300 ++++++++++++ tests/init_test.py | 99 + tests/logging.py | 0 tests/merge.py | 454 ++++ tests/option_test.py | 218 ++ tests/page.py | 641 ++++++ tests/pgpro560.py | 98 + tests/pgpro589.py | 80 + tests/ptrack.py | 1600 ++++++++++++++ tests/ptrack_clean.py | 253 +++ tests/ptrack_cluster.py | 268 +++ tests/ptrack_move_to_tablespace.py | 57 + tests/ptrack_recovery.py | 58 + tests/ptrack_truncate.py | 130 ++ tests/ptrack_vacuum.py | 152 ++ tests/ptrack_vacuum_bits_frozen.py | 136 ++ tests/ptrack_vacuum_bits_visibility.py | 67 + tests/ptrack_vacuum_full.py | 140 ++ tests/ptrack_vacuum_truncate.py | 142 ++ tests/replica.py | 293 +++ tests/restore_test.py | 1243 +++++++++++ tests/retention_test.py | 178 ++ tests/show_test.py | 203 ++ tests/validate_test.py | 1730 +++++++++++++++ travis/backup_restore.sh | 66 + win32build.pl | 240 +++ win32build96.pl | 240 +++ win32build_2.pl | 219 ++ 86 files changed, 34878 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 COPYRIGHT create mode 100644 Makefile create mode 100644 README.md create mode 100644 doit.cmd create mode 100644 doit96.cmd create mode 100644 gen_probackup_project.pl create mode 100644 msvs/pg_probackup.sln create mode 100644 msvs/template.pg_probackup.vcxproj create mode 100644 msvs/template.pg_probackup96.vcxproj create mode 100644 msvs/template.pg_probackup_2.vcxproj create mode 100644 src/archive.c create mode 100644 src/backup.c create mode 100644 src/catalog.c create mode 100644 src/configure.c create mode 100644 src/data.c create mode 100644 src/delete.c create mode 100644 src/dir.c create mode 100644 src/fetch.c create mode 100644 src/help.c create mode 100644 src/init.c create mode 100644 src/merge.c create mode 100644 src/parsexlog.c create mode 100644 src/pg_probackup.c create mode 100644 src/pg_probackup.h create mode 100644 src/restore.c create mode 100644 src/show.c create mode 100644 src/status.c create mode 100644 src/util.c create mode 100644 src/utils/json.c create mode 100644 src/utils/json.h create mode 100644 src/utils/logger.c create mode 100644 src/utils/logger.h create mode 100644 src/utils/parray.c create mode 100644 src/utils/parray.h create mode 100644 src/utils/pgut.c create mode 100644 src/utils/pgut.h create mode 100644 src/utils/thread.c create mode 100644 src/utils/thread.h create mode 100644 src/validate.c create mode 100644 tests/Readme.md create mode 100644 tests/__init__.py create mode 100644 tests/archive.py create mode 100644 tests/auth_test.py create mode 100644 tests/backup_test.py create mode 100644 tests/cfs_backup.py create mode 100644 tests/cfs_restore.py create mode 100644 tests/cfs_validate_backup.py create mode 100644 tests/compression.py create mode 100644 tests/delete_test.py create mode 100644 tests/delta.py create mode 100644 tests/exclude.py create mode 100644 tests/expected/option_help.out create mode 100644 tests/expected/option_version.out create mode 100644 tests/false_positive.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/cfs_helpers.py create mode 100644 tests/helpers/ptrack_helpers.py create mode 100644 tests/init_test.py create mode 100644 tests/logging.py create mode 100644 tests/merge.py create mode 100644 tests/option_test.py create mode 100644 tests/page.py create mode 100644 tests/pgpro560.py create mode 100644 tests/pgpro589.py create mode 100644 tests/ptrack.py create mode 100644 tests/ptrack_clean.py create mode 100644 tests/ptrack_cluster.py create mode 100644 tests/ptrack_move_to_tablespace.py create mode 100644 tests/ptrack_recovery.py create mode 100644 tests/ptrack_truncate.py create mode 100644 tests/ptrack_vacuum.py create mode 100644 tests/ptrack_vacuum_bits_frozen.py create mode 100644 tests/ptrack_vacuum_bits_visibility.py create mode 100644 tests/ptrack_vacuum_full.py create mode 100644 tests/ptrack_vacuum_truncate.py create mode 100644 tests/replica.py create mode 100644 tests/restore_test.py create mode 100644 tests/retention_test.py create mode 100644 tests/show_test.py create mode 100644 tests/validate_test.py create mode 100644 travis/backup_restore.sh create mode 100644 win32build.pl create mode 100644 win32build96.pl create mode 100644 win32build_2.pl diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..02d1512a --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# Object files +*.o + +# Libraries +*.lib +*.a + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.app + +# Dependencies +.deps + +# Binaries +/pg_probackup + +# Generated by test suite +/regression.diffs +/regression.out +/results +/env +/tests/__pycache__/ +/tests/helpers/__pycache__/ +/tests/tmp_dirs/ +/tests/*pyc +/tests/helpers/*pyc + +# Extra files +/src/datapagemap.c +/src/datapagemap.h +/src/logging.h +/src/receivelog.c +/src/receivelog.h +/src/streamutil.c +/src/streamutil.h +/src/xlogreader.c +/src/walmethods.c +/src/walmethods.h diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..35b49ec5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +sudo: required + +services: +- docker + +script: +- docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 00000000..49d70472 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,29 @@ +Copyright (c) 2015-2017, Postgres Professional +Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + +Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group +Portions Copyright (c) 1994, The Regents of the University of California + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION + (NTT) nor the names of its contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..56ad1b01 --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +PROGRAM = pg_probackup +OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ + src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \ + src/pg_probackup.o src/restore.o src/show.o src/status.o \ + src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ + src/xlogreader.o src/streamutil.o src/receivelog.o \ + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ + src/utils/json.o src/utils/thread.o src/merge.o + +EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h + +INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +# !USE_PGXS +else +subdir=contrib/pg_probackup +top_builddir=../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif # USE_PGXS + +ifeq ($(top_srcdir),../..) + ifeq ($(LN_S),ln -s) + srchome=$(top_srcdir)/.. + endif +else +srchome=$(top_srcdir) +endif + +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) +OBJS += src/walmethods.o +EXTRA_CLEAN += src/walmethods.c src/walmethods.h +INCLUDES += src/walmethods.h +endif + +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src +override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) +PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} + +all: checksrcdir $(INCLUDES); + +$(PROGRAM): $(OBJS) + +src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c + rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ +src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ +src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@ +src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ +src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@ +src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.h $@ +src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ +src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ + + +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) +src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ +src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ +endif + +ifeq ($(PORTNAME), aix) + CC=xlc_r +endif + +# This rule's only purpose is to give the user instructions on how to pass +# the path to PostgreSQL source tree to the makefile. +.PHONY: checksrcdir +checksrcdir: +ifndef top_srcdir + @echo "You must have PostgreSQL source tree available to compile." + @echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir" + @echo "variable: \"make top_srcdir=\"" + @exit 1 +endif diff --git a/README.md b/README.md new file mode 100644 index 00000000..1471d648 --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# pg_probackup + +`pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. + +The utility is compatible with: +* PostgreSQL 9.5, 9.6, 10; + +`PTRACK` backup support provided via following options: +* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) +* Postgres Pro Standard 9.5, 9.6 +* Postgres Pro Enterprise + +As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: +* Choosing between full and page-level incremental backups to speed up backup and recovery +* Implementing a single backup strategy for multi-server PostgreSQL clusters +* Automatic data consistency checks and on-demand backup validation without actual data recovery +* Managing backups in accordance with retention policy +* Running backup, restore, and validation processes on multiple parallel threads +* Storing backup data in a compressed state to save disk space +* Taking backups from a standby server to avoid extra load on the master server +* Extended logging settings +* Custom commands to simplify WAL log archiving + +To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. + +Using `pg_probackup`, you can take full or incremental backups: +* `Full` backups contain all the data files required to restore the database cluster from scratch. +* `Incremental` backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. `pg_probackup` supports the following modes of incremental backups: + * `PAGE` backup. In this mode, `pg_probackup` scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. + * `DELTA` backup. In this mode, `pg_probackup` read all data files in PGDATA directory and only those pages, that where changed since previous backup, are copied. Continuous archiving is not necessary for it to operate. Also this mode could impose read-only I/O pressure equal to `Full` backup. + * `PTRACK` backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special `PTRACK` bitmap for this relation. As one page requires just one bit in the `PTRACK` fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. + +Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following archiving strategies: +* `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. +* `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). + +## Limitations + +`pg_probackup` currently has the following limitations: +* Creating backups from a remote server is currently not supported. +* The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-block-size) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-wal-block-size) parameters and have the same major release number. +* Microsoft Windows operating system is not supported. +* Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately. + +## Installation and Setup +### Linux Installation +```shell +#DEB Ubuntu|Debian Packages +echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list +wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update +apt-get install pg-probackup-{10,9.6,9.5} + +#DEB-SRC Packages +echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ + /etc/apt/sources.list.d/pg_probackup.list +apt-get source pg-probackup-{10,9.6,9.5} + +#RPM Centos Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#RPM RHEL Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#RPM Oracle Linux Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#SRPM Packages +yumdownloader --source pg_probackup-{10,9.6,9.5} +``` + +To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: + +```shell +make USE_PGXS=1 PG_CONFIG= top_srcdir= +``` + +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). + +## Documentation + +Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). + +## Licence + +This module available under the same license as [PostgreSQL](https://www.postgresql.org/about/licence/). + +## Feedback + +Do not hesitate to post your issues, questions and new ideas at the [issues](https://github.com/postgrespro/pg_probackup/issues) page. + +## Authors + +Postgres Professional, Moscow, Russia. + +## Credits + +`pg_probackup` utility is based on `pg_arman`, that was originally written by NTT and then developed and maintained by Michael Paquier. \ No newline at end of file diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 00000000..b46e3b36 --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 00000000..94d242c9 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl new file mode 100644 index 00000000..3ea79e96 --- /dev/null +++ b/gen_probackup_project.pl @@ -0,0 +1,190 @@ +# -*-perl-*- hey - emacs - this is a perl file +BEGIN{ +use Cwd; +use File::Basename; + +my $pgsrc=""; +if (@ARGV==1) +{ + $pgsrc = shift @ARGV; + if($pgsrc == "--help"){ + print STDERR "Usage $0 pg-source-dir \n"; + print STDERR "Like this: \n"; + print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n"; + print STDERR "May be need input this before: \n"; + print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n"; + exit 1; + } +} +else +{ + use Cwd qw(abs_path); + my $path = dirname(abs_path($0)); + chdir($path); + chdir("../.."); + $pgsrc = cwd(); +} + +chdir("$pgsrc/src/tools/msvc"); +push(@INC, "$pgsrc/src/tools/msvc"); +chdir("../../..") if (-d "../msvc" && -d "../../../src"); + +} + +use Win32; +use Carp; +use strict; +use warnings; + + +use Project; +use Solution; +use File::Copy; +use Config; +use VSObjectFactory; +use List::Util qw(first); + +use Exporter; +our (@ISA, @EXPORT_OK); +@ISA = qw(Exporter); +@EXPORT_OK = qw(Mkvcbuild); + +my $solution; +my $libpgport; +my $libpgcommon; +my $libpgfeutils; +my $postgres; +my $libpq; +my @unlink_on_exit; + + +use lib "src/tools/msvc"; + +use Mkvcbuild; + +# if (-e "src/tools/msvc/buildenv.pl") +# { +# do "src/tools/msvc/buildenv.pl"; +# } +# elsif (-e "./buildenv.pl") +# { +# do "./buildenv.pl"; +# } + +# set up the project +our $config; +do "config_default.pl"; +do "config.pl" if (-f "src/tools/msvc/config.pl"); + +# my $vcver = Mkvcbuild::mkvcbuild($config); +my $vcver = build_pgprobackup($config); + +# check what sort of build we are doing + +my $bconf = $ENV{CONFIG} || "Release"; +my $msbflags = $ENV{MSBFLAGS} || ""; +my $buildwhat = $ARGV[1] || ""; +if (uc($ARGV[0]) eq 'DEBUG') +{ + $bconf = "Debug"; +} +elsif (uc($ARGV[0]) ne "RELEASE") +{ + $buildwhat = $ARGV[0] || ""; +} + +# ... and do it +system("msbuild pg_probackup.vcxproj /verbosity:normal $msbflags /p:Configuration=$bconf" ); + + +# report status + +my $status = $? >> 8; + +exit $status; + + + +sub build_pgprobackup +{ + our $config = shift; + + chdir('../../..') if (-d '../msvc' && -d '../../../src'); + die 'Must run from root or msvc directory' + unless (-d 'src/tools/msvc' && -d 'src'); + + # my $vsVersion = DetermineVisualStudioVersion(); + my $vsVersion = '12.00'; + + $solution = CreateSolution($vsVersion, $config); + + $libpq = $solution->AddProject('libpq', 'dll', 'interfaces', + 'src/interfaces/libpq'); + $libpgfeutils = $solution->AddProject('libpgfeutils', 'lib', 'misc'); + $libpgcommon = $solution->AddProject('libpgcommon', 'lib', 'misc'); + $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); + + #vvs test + my $probackup = + $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $probackup->AddFiles( + 'contrib/pg_probackup/src', + 'archive.c', + 'backup.c', + 'catalog.c', + 'configure.c', + 'data.c', + 'delete.c', + 'dir.c', + 'fetch.c', + 'help.c', + 'init.c', + 'parsexlog.c', + 'pg_probackup.c', + 'restore.c', + 'show.c', + 'status.c', + 'util.c', + 'validate.c' + ); + $probackup->AddFiles( + 'contrib/pg_probackup/src/utils', + 'json.c', + 'logger.c', + 'parray.c', + 'pgut.c', + 'thread.c' + ); + $probackup->AddFile('src/backend/access/transam/xlogreader.c'); + $probackup->AddFiles( + 'src/bin/pg_basebackup', + 'receivelog.c', + 'streamutil.c' + ); + + if (-e 'src/bin/pg_basebackup/walmethods.c') + { + $probackup->AddFile('src/bin/pg_basebackup/walmethods.c'); + } + + $probackup->AddFile('src/bin/pg_rewind/datapagemap.c'); + + $probackup->AddFile('src/interfaces/libpq/pthread-win32.c'); + + $probackup->AddIncludeDir('src/bin/pg_basebackup'); + $probackup->AddIncludeDir('src/bin/pg_rewind'); + $probackup->AddIncludeDir('src/interfaces/libpq'); + $probackup->AddIncludeDir('src'); + $probackup->AddIncludeDir('src/port'); + + $probackup->AddIncludeDir('contrib/pg_probackup'); + $probackup->AddIncludeDir('contrib/pg_probackup/src'); + $probackup->AddIncludeDir('contrib/pg_probackup/src/utils'); + + $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + $probackup->AddLibrary('ws2_32.lib'); + + $probackup->Save(); + return $solution->{vcver}; + +} diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 00000000..2df4b404 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 00000000..46a7b2c2 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,212 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 00000000..46e019ba --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,210 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 00000000..2fc101a4 --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/archive.c b/src/archive.c new file mode 100644 index 00000000..953a6877 --- /dev/null +++ b/src/archive.c @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * archive.c: - pg_probackup specific archive commands for archive backups. + * + * + * Portions Copyright (c) 2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +#include +#include + +/* + * pg_probackup specific archive command for archive backups + * set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup + * --wal-file-path %p --wal-file-name %f', to move backups into arclog_path. + * Where archlog_path is $BACKUP_PATH/wal/system_id. + * Currently it just copies wal files to the new location. + * TODO: Planned options: list the arclog content, + * compute and validate checksums. + */ +int +do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + int64 system_id; + pgBackupConfig *config; + bool is_compress = false; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + /* verify that archive-push --instance parameter is valid */ + config = readBackupCatalogConfigFile(); + system_id = get_system_identifier(current_dir); + + if (config->pgdata == NULL) + elog(ERROR, "cannot read pg_probackup.conf for this instance"); + + if(system_id != config->system_identifier) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have SYSTEM_ID = " INT64_FORMAT " instead of " INT64_FORMAT, + wal_file_name, instance_name, config->system_identifier, system_id); + + /* Create 'archlog_path' directory. Do nothing if it already exists. */ + dir_create_dir(arclog_path, DIR_PERMISSION); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + + if (compress_alg == PGLZ_COMPRESS) + elog(ERROR, "pglz compression is not supported"); + +#ifdef HAVE_LIBZ + if (compress_alg == ZLIB_COMPRESS) + is_compress = IsXLogFileName(wal_file_name); +#endif + + push_wal_file(absolute_wal_file_path, backup_wal_file_path, is_compress, + overwrite); + elog(INFO, "pg_probackup archive-push completed successfully"); + + return 0; +} + +/* + * pg_probackup specific restore command. + * Move files from arclog_path to pgdata/wal_file_path. + */ +int +do_archive_get(char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-get from %s to %s", + backup_wal_file_path, absolute_wal_file_path); + get_wal_file(backup_wal_file_path, absolute_wal_file_path); + elog(INFO, "pg_probackup archive-get completed successfully"); + + return 0; +} diff --git a/src/backup.c b/src/backup.c new file mode 100644 index 00000000..3aa36c98 --- /dev/null +++ b/src/backup.c @@ -0,0 +1,2701 @@ +/*------------------------------------------------------------------------- + * + * backup.c: backup DB cluster, archived WAL + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" +#include "datapagemap.h" +#include "libpq/pqsignal.h" +#include "pgtar.h" +#include "receivelog.h" +#include "storage/bufpage.h" +#include "streamutil.h" +#include "utils/thread.h" + +static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ +static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; +static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; + +/* + * How long we should wait for streaming end in seconds. + * Retreived as checkpoint_timeout + checkpoint_timeout * 0.1 + */ +static uint32 stream_stop_timeout = 0; +/* Time in which we started to wait for streaming end */ +static time_t stream_stop_begin = 0; + +const char *progname = "pg_probackup"; + +/* list of files contained in backup */ +static parray *backup_files_list = NULL; + +/* We need critical section for datapagemap_add() in case of using threads */ +static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * We need to wait end of WAL streaming before execute pg_stop_backup(). + */ +typedef struct +{ + const char *basedir; + PGconn *conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} StreamThreadArg; + +static pthread_t stream_thread; +static StreamThreadArg stream_thread_arg = {"", NULL, 1}; + +static int is_ptrack_enable = false; +bool is_ptrack_support = false; +bool is_checksum_enabled = false; +bool exclusive_backup = false; + +/* Backup connections */ +static PGconn *backup_conn = NULL; +static PGconn *master_conn = NULL; +static PGconn *backup_conn_replication = NULL; + +/* PostgreSQL server version from "backup_conn" */ +static int server_version = 0; +static char server_version_str[100] = ""; + +/* Is pg_start_backup() was executed */ +static bool backup_in_progress = false; +/* Is pg_stop_backup() was sent */ +static bool pg_stop_backup_is_sent = false; + +/* + * Backup routines + */ +static void backup_cleanup(bool fatal, void *userdata); +static void backup_disconnect(bool fatal, void *userdata); + +static void *backup_files(void *arg); +static void *remote_backup_files(void *arg); + +static void do_backup_instance(void); + +static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); +static void pg_switch_wal(PGconn *conn); +static void pg_stop_backup(pgBackup *backup); +static int checkpoint_timeout(void); + +//static void backup_list_file(parray *files, const char *root, ) +static void parse_backup_filelist_filenames(parray *files, const char *root); +static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment); +static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); +static void make_pagemap_from_ptrack(parray *files); +static void *StreamLog(void *arg); + +static void get_remote_pgdata_filelist(parray *files); +static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); +static void remote_copy_file(PGconn *conn, pgFile* file); + +/* Ptrack functions */ +static void pg_ptrack_clear(void); +static bool pg_ptrack_support(void); +static bool pg_ptrack_enable(void); +static bool pg_checksum_enable(void); +static bool pg_is_in_recovery(void); +static bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid); +static char *pg_ptrack_get_and_clear(Oid tablespace_oid, + Oid db_oid, + Oid rel_oid, + size_t *result_size); +static XLogRecPtr get_last_ptrack_lsn(void); + +/* Check functions */ +static void check_server_version(void); +static void check_system_identifiers(void); +static void confirm_block_size(const char *name, int blcksz); +static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); + +#define disconnect_and_exit(code) \ + { \ + if (conn != NULL) PQfinish(conn); \ + exit(code); \ + } + +/* Fill "files" with data about all the files to backup */ +static void +get_remote_pgdata_filelist(parray *files) +{ + PGresult *res; + int resultStatus; + int i; + + backup_conn_replication = pgut_connect_replication(pgut_dbname); + + if (PQsendQuery(backup_conn_replication, "FILE_BACKUP FILELIST") == 0) + elog(ERROR,"%s: could not send replication command \"%s\": %s", + PROGRAM_NAME, "FILE_BACKUP", PQerrorMessage(backup_conn_replication)); + + res = PQgetResult(backup_conn_replication); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + resultStatus = PQresultStatus(res); + PQclear(res); + elog(ERROR, "cannot start getting FILE_BACKUP filelist: %s, result_status %d", + PQerrorMessage(backup_conn_replication), resultStatus); + } + + if (PQntuples(res) < 1) + elog(ERROR, "%s: no data returned from server", PROGRAM_NAME); + + for (i = 0; i < PQntuples(res); i++) + { + ReceiveFileList(files, backup_conn_replication, res, i); + } + + res = PQgetResult(backup_conn_replication); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(ERROR, "%s: final receive failed: %s", + PROGRAM_NAME, PQerrorMessage(backup_conn_replication)); + } + + PQfinish(backup_conn_replication); +} + +/* + * workhorse for get_remote_pgdata_filelist(). + * Parse received message into pgFile structure. + */ +static void +ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) +{ + char filename[MAXPGPATH]; + pgoff_t current_len_left = 0; + bool basetablespace; + char *copybuf = NULL; + pgFile *pgfile; + + /* What for do we need this basetablespace field?? */ + basetablespace = PQgetisnull(res, rownum, 0); + if (basetablespace) + elog(LOG,"basetablespace"); + else + elog(LOG, "basetablespace %s", PQgetvalue(res, rownum, 1)); + + res = PQgetResult(conn); + + if (PQresultStatus(res) != PGRES_COPY_OUT) + elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(conn)); + + while (1) + { + int r; + int filemode; + + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + r = PQgetCopyData(conn, ©buf, 0); + + if (r == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); + + /* end of copy */ + if (r == -1) + break; + + /* This must be the header for a new file */ + if (r != 512) + elog(ERROR, "Invalid tar block header size: %d\n", r); + + current_len_left = read_tar_number(©buf[124], 12); + + /* Set permissions on the file */ + filemode = read_tar_number(©buf[100], 8); + + /* First part of header is zero terminated filename */ + snprintf(filename, sizeof(filename), "%s", copybuf); + + pgfile = pgFileInit(filename); + pgfile->size = current_len_left; + pgfile->mode |= filemode; + + if (filename[strlen(filename) - 1] == '/') + { + /* Symbolic link or directory has size zero */ + Assert (pgfile->size == 0); + /* Ends in a slash means directory or symlink to directory */ + if (copybuf[156] == '5') + { + /* Directory */ + pgfile->mode |= S_IFDIR; + } + else if (copybuf[156] == '2') + { + /* Symlink */ +#ifndef WIN32 + pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif + } + else + elog(ERROR, "Unrecognized link indicator \"%c\"\n", + copybuf[156]); + } + else + { + /* regular file */ + pgfile->mode |= S_IFREG; + } + + parray_append(files, pgfile); + } + + if (copybuf != NULL) + PQfreemem(copybuf); +} + +/* read one file via replication protocol + * and write it to the destination subdir in 'backup_path' */ +static void +remote_copy_file(PGconn *conn, pgFile* file) +{ + PGresult *res; + char *copybuf = NULL; + char buf[32768]; + FILE *out; + char database_path[MAXPGPATH]; + char to_path[MAXPGPATH]; + bool skip_padding = false; + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + join_path_components(to_path, database_path, file->path); + + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + INIT_CRC32C(file->crc); + + /* read from stream and write to backup file */ + while (1) + { + int row_length; + int errno_tmp; + int write_buffer_size = 0; + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + row_length = PQgetCopyData(conn, ©buf, 0); + + if (row_length == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); + + if (row_length == -1) + break; + + if (!skip_padding) + { + write_buffer_size = Min(row_length, sizeof(buf)); + memcpy(buf, copybuf, write_buffer_size); + COMP_CRC32C(file->crc, buf, write_buffer_size); + + /* TODO calc checksum*/ + if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) + { + errno_tmp = errno; + /* oops */ + FIN_CRC32C(file->crc); + fclose(out); + PQfinish(conn); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + file->read_size += write_buffer_size; + } + if (file->read_size >= file->size) + { + skip_padding = true; + } + } + + res = PQgetResult(conn); + + /* File is not found. That's normal. */ + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); + } + + file->write_size = (int64) file->read_size; + FIN_CRC32C(file->crc); + + fclose(out); +} + +/* + * Take a remote backup of the PGDATA at a file level. + * Copy all directories and files listed in backup_files_list. + */ +static void * +remote_backup_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + PGconn *file_backup_conn = NULL; + + for (i = 0; i < n_backup_files_list; i++) + { + char *query_str; + PGresult *res; + char *copybuf = NULL; + pgFile *file; + int row_length; + + file = (pgFile *) parray_get(arguments->files_list, i); + + /* We have already copied all directories */ + if (S_ISDIR(file->mode)) + continue; + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + file_backup_conn = pgut_connect_replication(pgut_dbname); + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + query_str = psprintf("FILE_BACKUP FILEPATH '%s'",file->path); + + if (PQsendQuery(file_backup_conn, query_str) == 0) + elog(ERROR,"%s: could not send replication command \"%s\": %s", + PROGRAM_NAME, query_str, PQerrorMessage(file_backup_conn)); + + res = PQgetResult(file_backup_conn); + + /* File is not found. That's normal. */ + if (PQresultStatus(res) == PGRES_COMMAND_OK) + { + PQclear(res); + PQfinish(file_backup_conn); + continue; + } + + if (PQresultStatus(res) != PGRES_COPY_OUT) + { + PQclear(res); + PQfinish(file_backup_conn); + elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(file_backup_conn)); + } + + /* read the header of the file */ + row_length = PQgetCopyData(file_backup_conn, ©buf, 0); + + if (row_length == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(file_backup_conn)); + + /* end of copy TODO handle it */ + if (row_length == -1) + elog(ERROR, "Unexpected end of COPY data"); + + if(row_length != 512) + elog(ERROR, "Invalid tar block header size: %d\n", row_length); + file->size = read_tar_number(©buf[124], 12); + + /* receive the data from stream and write to backup file */ + remote_copy_file(file_backup_conn, file); + + elog(VERBOSE, "File \"%s\". Copied " INT64_FORMAT " bytes", + file->path, file->write_size); + PQfinish(file_backup_conn); + } + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Take a backup of a single postgresql instance. + * Move files from 'pgdata' to a subdirectory in 'backup_path'. + */ +static void +do_backup_instance(void) +{ + int i; + char database_path[MAXPGPATH]; + char dst_backup_path[MAXPGPATH]; + char label[1024]; + XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; + + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + backup_files_arg *threads_args; + bool backup_isok = true; + + pgBackup *prev_backup = NULL; + parray *prev_backup_filelist = NULL; + + elog(LOG, "Database backup start"); + + /* Initialize size summary */ + current.data_bytes = 0; + + /* Obtain current timeline */ + if (is_remote_backup) + { + char *sysidentifier; + TimeLineID starttli; + XLogRecPtr startpos; + + backup_conn_replication = pgut_connect_replication(pgut_dbname); + + /* Check replication prorocol connection */ + if (!RunIdentifySystem(backup_conn_replication, &sysidentifier, &starttli, &startpos, NULL)) + elog(ERROR, "Failed to send command for remote backup"); + +// TODO implement the check +// if (&sysidentifier != system_identifier) +// elog(ERROR, "Backup data directory was initialized for system id %ld, but target backup directory system id is %ld", +// system_identifier, sysidentifier); + + current.tli = starttli; + + PQfinish(backup_conn_replication); + } + else + current.tli = get_current_timeline(false); + + /* + * In incremental backup mode ensure that already-validated + * backup on current timeline exists and get its filelist. + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK || + current.backup_mode == BACKUP_MODE_DIFF_DELTA) + { + parray *backup_list; + char prev_backup_filelist_path[MAXPGPATH]; + + /* get list of backups already taken */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + prev_backup = catalog_get_last_data_backup(backup_list, current.tli); + if (prev_backup == NULL) + elog(ERROR, "Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one."); + parray_free(backup_list); + + pgBackupGetPath(prev_backup, prev_backup_filelist_path, + lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); + /* Files of previous backup needed by DELTA backup */ + prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path); + + /* If lsn is not NULL, only pages with higher lsn will be copied. */ + prev_backup_start_lsn = prev_backup->start_lsn; + current.parent_backup = prev_backup->start_time; + + pgBackupWriteBackupControlFile(¤t); + } + + /* + * It`s illegal to take PTRACK backup if LSN from ptrack_control() is not equal to + * stort_backup LSN of previous backup + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(); + + if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) + { + elog(ERROR, "LSN from ptrack_control " UINT64_FORMAT " differs from STOP LSN of previous backup " + UINT64_FORMAT ".\n" + "Create new full backup before an incremental one.", + ptrack_lsn, prev_backup->stop_lsn); + } + } + + /* Clear ptrack files for FULL and PAGE backup */ + if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && is_ptrack_enable) + pg_ptrack_clear(); + + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time); + strncat(label, " with pg_probackup", lengthof(label) - + strlen(" with pg_probackup")); + pg_start_backup(label, smooth_checkpoint, ¤t); + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + /* start stream replication */ + if (stream_wal) + { + join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); + dir_create_dir(dst_backup_path, DIR_PERMISSION); + + stream_thread_arg.basedir = dst_backup_path; + + /* + * Connect in replication mode to the server. + */ + stream_thread_arg.conn = pgut_connect_replication(pgut_dbname); + + if (!CheckServerVersionForStreaming(stream_thread_arg.conn)) + { + PQfinish(stream_thread_arg.conn); + /* + * Error message already written in CheckServerVersionForStreaming(). + * There's no hope of recovering from a version mismatch, so don't + * retry. + */ + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* + * Identify server, obtaining start LSN position and current timeline ID + * at the same time, necessary if not valid data can be found in the + * existing output directory. + */ + if (!RunIdentifySystem(stream_thread_arg.conn, NULL, NULL, NULL, NULL)) + { + PQfinish(stream_thread_arg.conn); + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* By default there are some error */ + stream_thread_arg.ret = 1; + + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); + } + + /* initialize backup list */ + backup_files_list = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + if (is_remote_backup) + get_remote_pgdata_filelist(backup_files_list); + else + dir_list_file(backup_files_list, pgdata, true, true, false); + + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + + /* Extract information about files in backup_list parsing their names:*/ + parse_backup_filelist_filenames(backup_files_list, pgdata); + + if (current.backup_mode != BACKUP_MODE_FULL) + { + elog(LOG, "current_tli:%X", current.tli); + elog(LOG, "prev_backup->start_lsn: %X/%X", + (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn)); + elog(LOG, "current.start_lsn: %X/%X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + } + + /* + * Build page mapping in incremental mode. + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + { + /* + * Build the page map. Obtain information about changed pages + * reading WAL segments present in archives up to the point + * where this backup has started. + */ + extractPageMap(arclog_path, current.tli, xlog_seg_size, + prev_backup->start_lsn, current.start_lsn, + /* + * For backup from master wait for previous segment. + * For backup from replica wait for current segment. + */ + !current.from_replica, backup_files_list); + } + else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + /* + * Build the page map from ptrack information. + */ + make_pagemap_from_ptrack(backup_files_list); + } + + /* + * Make directories before backup and setup threads at the same time + */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + /* if the entry was a directory, create it in the backup */ + if (S_ISDIR(file->mode)) + { + char dirpath[MAXPGPATH]; + char *dir_name; + char database_path[MAXPGPATH]; + + if (!is_remote_backup) + dir_name = GetRelativePath(file->path, pgdata); + else + dir_name = file->path; + + elog(VERBOSE, "Create directory \"%s\"", dir_name); + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + join_path_components(dirpath, database_path, dir_name); + dir_create_dir(dirpath, DIR_PERMISSION); + } + + /* setup threads */ + pg_atomic_clear_flag(&file->lock); + } + + /* Sort by size for load balancing */ + parray_qsort(backup_files_list, pgFileCompareSize); + /* Sort the array for binary search */ + if (prev_backup_filelist) + parray_qsort(prev_backup_filelist, pgFileComparePath); + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); + + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + arg->from_root = pgdata; + arg->to_root = database_path; + arg->files_list = backup_files_list; + arg->prev_filelist = prev_backup_filelist; + arg->prev_start_lsn = prev_backup_start_lsn; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; + /* By default there are some error */ + arg->ret = 1; + } + + /* Run threads */ + elog(LOG, "Start transfering data files"); + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + elog(VERBOSE, "Start thread num: %i", i); + + if (!is_remote_backup) + pthread_create(&threads[i], NULL, backup_files, arg); + else + pthread_create(&threads[i], NULL, remote_backup_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + backup_isok = false; + } + if (backup_isok) + elog(LOG, "Data files are transfered"); + else + elog(ERROR, "Data files transferring failed"); + + /* clean previous backup file list */ + if (prev_backup_filelist) + { + parray_walk(prev_backup_filelist, pgFileFree); + parray_free(prev_backup_filelist); + } + + /* Notify end of backup */ + pg_stop_backup(¤t); + + /* Add archived xlog files into the list of files of this backup */ + if (stream_wal) + { + parray *xlog_files_list; + char pg_xlog_path[MAXPGPATH]; + + /* Scan backup PG_XLOG_DIR */ + xlog_files_list = parray_new(); + join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + + for (i = 0; i < parray_num(xlog_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + + if (S_ISREG(file->mode)) + calc_file_checksum(file); + /* Remove file path root prefix*/ + if (strstr(file->path, database_path) == file->path) + { + char *ptr = file->path; + + file->path = pstrdup(GetRelativePath(ptr, database_path)); + free(ptr); + } + } + + /* Add xlog files into the list of backed up files */ + parray_concat(backup_files_list, xlog_files_list); + parray_free(xlog_files_list); + } + + /* Print the list of files to backup catalog */ + pgBackupWriteFileList(¤t, backup_files_list, pgdata); + + /* Compute summary of size of regular files in the backup */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + if (S_ISDIR(file->mode)) + current.data_bytes += 4096; + + /* Count the amount of the data actually copied */ + if (S_ISREG(file->mode)) + current.data_bytes += file->write_size; + } + + parray_walk(backup_files_list, pgFileFree); + parray_free(backup_files_list); + backup_files_list = NULL; +} + +/* + * Entry point of pg_probackup BACKUP subcommand. + */ +int +do_backup(time_t start_time) +{ + + /* PGDATA and BACKUP_MODE are always required */ + if (pgdata == NULL) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); + if (current.backup_mode == BACKUP_MODE_INVALID) + elog(ERROR, "required parameter not specified: BACKUP_MODE " + "(-b, --backup-mode)"); + + /* Create connection for PostgreSQL */ + backup_conn = pgut_connect(pgut_dbname); + pgut_atexit_push(backup_disconnect, NULL); + + current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + +#if PG_VERSION_NUM >= 110000 + if (!RetrieveWalSegSize(backup_conn)) + elog(ERROR, "Failed to retreive wal_segment_size"); +#endif + + current.compress_alg = compress_alg; + current.compress_level = compress_level; + + /* Confirm data block size and xlog block size are compatible */ + confirm_block_size("block_size", BLCKSZ); + confirm_block_size("wal_block_size", XLOG_BLCKSZ); + + current.from_replica = pg_is_in_recovery(); + + /* Confirm that this server version is supported */ + check_server_version(); + + /* TODO fix it for remote backup*/ + if (!is_remote_backup) + current.checksum_version = get_data_checksum_version(true); + + is_checksum_enabled = pg_checksum_enable(); + + if (is_checksum_enabled) + elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " + "Data block corruption will be detected"); + else + elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " + "pg_probackup have no way to detect data block corruption without them. " + "Reinitialize PGDATA with option '--data-checksums'."); + + StrNCpy(current.server_version, server_version_str, + sizeof(current.server_version)); + current.stream = stream_wal; + + is_ptrack_support = pg_ptrack_support(); + if (is_ptrack_support) + { + is_ptrack_enable = pg_ptrack_enable(); + } + + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + if (!is_ptrack_support) + elog(ERROR, "This PostgreSQL instance does not support ptrack"); + else + { + if(!is_ptrack_enable) + elog(ERROR, "Ptrack is disabled"); + } + } + + if (current.from_replica) + { + /* Check master connection options */ + if (master_host == NULL) + elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); + + /* Create connection to master server */ + master_conn = pgut_connect_extended(master_host, master_port, master_db, master_user); + } + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + */ + /* TODO fix it for remote backup */ + if (!is_remote_backup) + check_system_identifiers(); + + + /* Start backup. Update backup status. */ + current.status = BACKUP_STATUS_RUNNING; + current.start_time = start_time; + + /* Create backup directory and BACKUP_CONTROL_FILE */ + if (pgBackupCreateDir(¤t)) + elog(ERROR, "cannot create backup directory"); + pgBackupWriteBackupControlFile(¤t); + + elog(LOG, "Backup destination is initialized"); + + /* set the error processing function for the backup process */ + pgut_atexit_push(backup_cleanup, NULL); + + /* backup data */ + do_backup_instance(); + pgut_atexit_pop(backup_cleanup, NULL); + + /* compute size of wal files of this backup stored in the archive */ + if (!current.stream) + { + current.wal_bytes = xlog_seg_size * + (current.stop_lsn / xlog_seg_size - + current.start_lsn / xlog_seg_size + 1); + } + + /* Backup is done. Update backup status */ + current.end_time = time(NULL); + current.status = BACKUP_STATUS_DONE; + pgBackupWriteBackupControlFile(¤t); + + //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", + // current.data_bytes); + + pgBackupValidate(¤t); + + elog(INFO, "Backup %s completed", base36enc(current.start_time)); + + /* + * After successfil backup completion remove backups + * which are expired according to retention policies + */ + if (delete_expired || delete_wal) + do_retention_purge(); + + return 0; +} + +/* + * Confirm that this server version is supported + */ +static void +check_server_version(void) +{ + PGresult *res; + + /* confirm server version */ + server_version = PQserverVersion(backup_conn); + + if (server_version == 0) + elog(ERROR, "Unknown server version %d", server_version); + + if (server_version < 100000) + sprintf(server_version_str, "%d.%d", + server_version / 10000, + (server_version / 100) % 100); + else + sprintf(server_version_str, "%d", + server_version / 10000); + + if (server_version < 90500) + elog(ERROR, + "server version is %s, must be %s or higher", + server_version_str, "9.5"); + + if (current.from_replica && server_version < 90600) + elog(ERROR, + "server version is %s, must be %s or higher for backup from replica", + server_version_str, "9.6"); + + res = pgut_execute_extended(backup_conn, "SELECT pgpro_edition()", + 0, NULL, true, true); + + /* + * Check major version of connected PostgreSQL and major version of + * compiled PostgreSQL. + */ +#ifdef PGPRO_VERSION + if (PQresultStatus(res) == PGRES_FATAL_ERROR) + /* It seems we connected to PostgreSQL (not Postgres Pro) */ + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with PostgreSQL %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str); + else if (strcmp(server_version_str, PG_MAJORVERSION) != 0 && + strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, + server_version_str, PQgetvalue(res, 0, 0)); +#else + if (PQresultStatus(res) != PGRES_FATAL_ERROR) + /* It seems we connected to Postgres Pro (not PostgreSQL) */ + elog(ERROR, "%s was built with PostgreSQL %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, + server_version_str, PQgetvalue(res, 0, 0)); + else if (strcmp(server_version_str, PG_MAJORVERSION) != 0) + elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", + PROGRAM_NAME, PG_MAJORVERSION, server_version_str); +#endif + + PQclear(res); + + /* Do exclusive backup only for PostgreSQL 9.5 */ + exclusive_backup = server_version < 90600 || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK; +} + +/* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + * All system identifiers must be equal. + */ +static void +check_system_identifiers(void) +{ + uint64 system_id_conn; + uint64 system_id_pgdata; + + system_id_pgdata = get_system_identifier(pgdata); + system_id_conn = get_remote_system_identifier(backup_conn); + + if (system_id_conn != system_identifier) + elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT + ", but connected instance system id is " UINT64_FORMAT, + system_identifier, system_id_conn); + if (system_id_pgdata != system_identifier) + elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT + ", but target backup directory system id is " UINT64_FORMAT, + system_identifier, system_id_pgdata); +} + +/* + * Ensure that target backup database is initialized with + * compatible settings. Currently check BLCKSZ and XLOG_BLCKSZ. + */ +static void +confirm_block_size(const char *name, int blcksz) +{ + PGresult *res; + char *endp; + int block_size; + + res = pgut_execute(backup_conn, "SELECT pg_catalog.current_setting($1)", 1, &name); + if (PQntuples(res) != 1 || PQnfields(res) != 1) + elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); + + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); + if ((endp && *endp) || block_size != blcksz) + elog(ERROR, + "%s(%d) is not compatible(%d expected)", + name, block_size, blcksz); + + PQclear(res); +} + +/* + * Notify start of backup to PostgreSQL server. + */ +static void +pg_start_backup(const char *label, bool smooth, pgBackup *backup) +{ + PGresult *res; + const char *params[2]; + uint32 xlogid; + uint32 xrecoff; + PGconn *conn; + + params[0] = label; + + /* For replica we call pg_start_backup() on master */ + conn = (backup->from_replica) ? master_conn : backup_conn; + + /* 2nd argument is 'fast'*/ + params[1] = smooth ? "false" : "true"; + if (!exclusive_backup) + res = pgut_execute(conn, + "SELECT pg_catalog.pg_start_backup($1, $2, false)", + 2, + params); + else + res = pgut_execute(conn, + "SELECT pg_catalog.pg_start_backup($1, $2)", + 2, + params); + + /* + * Set flag that pg_start_backup() was called. If an error will happen it + * is necessary to call pg_stop_backup() in backup_cleanup(). + */ + backup_in_progress = true; + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + PQclear(res); + + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + /* + * Switch to a new WAL segment. It is necessary to get archived WAL + * segment, which includes start LSN of current backup. + */ + pg_switch_wal(conn); + + if (!stream_wal) + { + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + /* In PAGE mode wait for current segment... */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + wait_wal_lsn(backup->start_lsn, false); + /* ...for others wait for previous segment */ + else + wait_wal_lsn(backup->start_lsn, true); + } + + /* Wait for start_lsn to be replayed by replica */ + if (backup->from_replica) + wait_replica_wal_lsn(backup->start_lsn, true); +} + +/* + * Switch to a new WAL segment. It should be called only for master. + */ +static void +pg_switch_wal(PGconn *conn) +{ + PGresult *res; + + /* Remove annoying NOTICE messages generated by backend */ + res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); + PQclear(res); + + if (server_version >= 100000) + res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_wal()", 0, NULL); + else + res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_xlog()", 0, NULL); + + PQclear(res); +} + +/* + * Check if the instance supports ptrack + * TODO Maybe we should rather check ptrack_version()? + */ +static bool +pg_ptrack_support(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, + "SELECT proname FROM pg_proc WHERE proname='ptrack_version'", + 0, NULL); + if (PQntuples(res_db) == 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + + res_db = pgut_execute(backup_conn, + "SELECT pg_catalog.ptrack_version()", + 0, NULL); + if (PQntuples(res_db) == 0) + { + PQclear(res_db); + return false; + } + + /* Now we support only ptrack versions upper than 1.5 */ + if (strcmp(PQgetvalue(res_db, 0, 0), "1.5") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0) + { + elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", PQgetvalue(res_db, 0, 0)); + PQclear(res_db); + return false; + } + + PQclear(res_db); + return true; +} + +/* Check if ptrack is enabled in target instance */ +static bool +pg_ptrack_enable(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show ptrack_enable", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* Check if ptrack is enabled in target instance */ +static bool +pg_checksum_enable(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show data_checksums", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* Check if target instance is replica */ +static bool +pg_is_in_recovery(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "SELECT pg_catalog.pg_is_in_recovery()", 0, NULL); + + if (PQgetvalue(res_db, 0, 0)[0] == 't') + { + PQclear(res_db); + return true; + } + PQclear(res_db); + return false; +} + +/* Clear ptrack files in all databases of the instance we connected to */ +static void +pg_ptrack_clear(void) +{ + PGresult *res_db, + *res; + const char *dbname; + int i; + Oid dbOid, tblspcOid; + char *params[2]; + + params[0] = palloc(64); + params[1] = palloc(64); + res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + 0, NULL); + + for(i = 0; i < PQntuples(res_db); i++) + { + PGconn *tmp_conn; + + dbname = PQgetvalue(res_db, i, 0); + if (strcmp(dbname, "template0") == 0) + continue; + + dbOid = atoi(PQgetvalue(res_db, i, 1)); + tblspcOid = atoi(PQgetvalue(res_db, i, 2)); + + tmp_conn = pgut_connect(dbname); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + PQclear(res); + + pgut_disconnect(tmp_conn); + } + + pfree(params[0]); + pfree(params[1]); + PQclear(res_db); +} + +static bool +pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid) +{ + char *params[2]; + char *dbname; + PGresult *res_db; + PGresult *res; + bool result; + + params[0] = palloc(64); + params[1] = palloc(64); + + sprintf(params[0], "%i", dbOid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return false; + + dbname = PQgetvalue(res_db, 0, 0); + + /* Always backup all files from template0 database */ + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return true; + } + PQclear(res_db); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot perform pg_ptrack_get_and_clear_db()"); + + if (!parse_bool(PQgetvalue(res, 0, 0), &result)) + elog(ERROR, + "result of pg_ptrack_get_and_clear_db() is invalid: %s", + PQgetvalue(res, 0, 0)); + + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* Read and clear ptrack files of the target relation. + * Result is a bytea ptrack map of all segments of the target relation. + * case 1: we know a tablespace_oid, db_oid, and rel_filenode + * case 2: we know db_oid and rel_filenode (no tablespace_oid, because file in pg_default) + * case 3: we know only rel_filenode (because file in pg_global) + */ +static char * +pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, + size_t *result_size) +{ + PGconn *tmp_conn; + PGresult *res_db, + *res; + char *params[2]; + char *result; + char *val; + + params[0] = palloc(64); + params[1] = palloc(64); + + /* regular file (not in directory 'global') */ + if (db_oid != 0) + { + char *dbname; + + sprintf(params[0], "%i", db_oid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return NULL; + + dbname = PQgetvalue(res_db, 0, 0); + + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return NULL; + } + + tmp_conn = pgut_connect(dbname); + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from database \"%s\" by tablespace oid %u and relation oid %u", + dbname, tablespace_oid, rel_filenode); + PQclear(res_db); + pgut_disconnect(tmp_conn); + } + /* file in directory 'global' */ + else + { + /* + * execute ptrack_get_and_clear for relation in pg_global + * Use backup_conn, cause we can do it from any database. + */ + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from pg_global tablespace and relation oid %u", + rel_filenode); + } + + val = PQgetvalue(res, 0, 0); + + /* TODO Now pg_ptrack_get_and_clear() returns bytea ending with \x. + * It should be fixed in future ptrack releases, but till then we + * can parse it. + */ + if (strcmp("x", val+1) == 0) + { + /* Ptrack file is missing */ + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* + * Wait for target 'lsn'. + * + * If current backup started in archive mode wait for 'lsn' to be archived in + * archive 'wal' directory with WAL segment file. + * If current backup started in stream mode wait for 'lsn' to be streamed in + * 'pg_wal' directory. + * + * If 'wait_prev_segment' wait for previous segment. + */ +static void +wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment) +{ + TimeLineID tli; + XLogSegNo targetSegNo; + char wal_dir[MAXPGPATH], + wal_segment_path[MAXPGPATH]; + char wal_segment[MAXFNAMELEN]; + bool file_exists = false; + uint32 try_count = 0, + timeout; + +#ifdef HAVE_LIBZ + char gz_wal_segment_path[MAXPGPATH]; +#endif + + tli = get_current_timeline(false); + + /* Compute the name of the WAL file containig requested LSN */ + GetXLogSegNo(lsn, targetSegNo, xlog_seg_size); + if (wait_prev_segment) + targetSegNo--; + GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size); + + if (stream_wal) + { + pgBackupGetPath2(¤t, wal_dir, lengthof(wal_dir), + DATABASE_DIR, PG_XLOG_DIR); + join_path_components(wal_segment_path, wal_dir, wal_segment); + + timeout = (uint32) checkpoint_timeout(); + timeout = timeout + timeout * 0.1; + } + else + { + join_path_components(wal_segment_path, arclog_path, wal_segment); + timeout = archive_timeout; + } + + if (wait_prev_segment) + elog(LOG, "Looking for segment: %s", wal_segment); + else + elog(LOG, "Looking for LSN: %X/%X in segment: %s", (uint32) (lsn >> 32), (uint32) lsn, wal_segment); + +#ifdef HAVE_LIBZ + snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", + wal_segment_path); +#endif + + /* Wait until target LSN is archived or streamed */ + while (true) + { + if (!file_exists) + { + file_exists = fileExists(wal_segment_path); + + /* Try to find compressed WAL file */ + if (!file_exists) + { +#ifdef HAVE_LIBZ + file_exists = fileExists(gz_wal_segment_path); + if (file_exists) + elog(LOG, "Found compressed WAL segment: %s", wal_segment_path); +#endif + } + else + elog(LOG, "Found WAL segment: %s", wal_segment_path); + } + + if (file_exists) + { + /* Do not check LSN for previous WAL segment */ + if (wait_prev_segment) + return; + + /* + * A WAL segment found. Check LSN on it. + */ + if ((stream_wal && wal_contains_lsn(wal_dir, lsn, tli, + xlog_seg_size)) || + (!stream_wal && wal_contains_lsn(arclog_path, lsn, tli, + xlog_seg_size))) + /* Target LSN was found */ + { + elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); + return; + } + } + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for WAL archiving"); + try_count++; + + /* Inform user if WAL segment is absent in first attempt */ + if (try_count == 1) + { + if (wait_prev_segment) + elog(INFO, "Wait for WAL segment %s to be archived", + wal_segment_path); + else + elog(INFO, "Wait for LSN %X/%X in archived WAL segment %s", + (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); + } + + if (timeout > 0 && try_count > timeout) + { + if (file_exists) + elog(ERROR, "WAL segment %s was archived, " + "but target LSN %X/%X could not be archived in %d seconds", + wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); + /* If WAL segment doesn't exist or we wait for previous segment */ + else + elog(ERROR, + "Switched WAL segment %s could not be archived in %d seconds", + wal_segment, timeout); + } + } +} + +/* + * Wait for target 'lsn' on replica instance from master. + */ +static void +wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) +{ + uint32 try_count = 0; + + while (true) + { + PGresult *res; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr replica_lsn; + + /* + * For lsn from pg_start_backup() we need it to be replayed on replica's + * data. + */ + if (is_start_backup) + { + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_replay_lsn()", + 0, NULL); + else + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_replay_location()", + 0, NULL); + } + /* + * For lsn from pg_stop_backup() we need it only to be received by + * replica and fsync()'ed on WAL segment. + */ + else + { + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_receive_lsn()", + 0, NULL); + else + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_receive_location()", + 0, NULL); + } + + /* Extract timeline and LSN from result */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + replica_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + PQclear(res); + + /* target lsn was replicated */ + if (replica_lsn >= lsn) + break; + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for target LSN"); + try_count++; + + /* Inform user if target lsn is absent in first attempt */ + if (try_count == 1) + elog(INFO, "Wait for target LSN %X/%X to be received by replica", + (uint32) (lsn >> 32), (uint32) lsn); + + if (replica_timeout > 0 && try_count > replica_timeout) + elog(ERROR, "Target LSN %X/%X could not be recevied by replica " + "in %d seconds", + (uint32) (lsn >> 32), (uint32) lsn, + replica_timeout); + } +} + +/* + * Notify end of backup to PostgreSQL server. + */ +static void +pg_stop_backup(pgBackup *backup) +{ + PGconn *conn; + PGresult *res; + PGresult *tablespace_map_content = NULL; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr restore_lsn = InvalidXLogRecPtr; + int pg_stop_backup_timeout = 0; + char path[MAXPGPATH]; + char backup_label[MAXPGPATH]; + FILE *fp; + pgFile *file; + size_t len; + char *val = NULL; + char *stop_backup_query = NULL; + + /* + * We will use this values if there are no transactions between start_lsn + * and stop_lsn. + */ + time_t recovery_time; + TransactionId recovery_xid; + + if (!backup_in_progress) + elog(ERROR, "backup is not in progress"); + + /* For replica we call pg_stop_backup() on master */ + conn = (current.from_replica) ? master_conn : backup_conn; + + /* Remove annoying NOTICE messages generated by backend */ + res = pgut_execute(conn, "SET client_min_messages = warning;", + 0, NULL); + PQclear(res); + + /* Create restore point */ + if (backup != NULL) + { + const char *params[1]; + char name[1024]; + + if (!current.from_replica) + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + base36enc(backup->start_time)); + else + snprintf(name, lengthof(name), "pg_probackup, backup_id %s. Replica Backup", + base36enc(backup->start_time)); + params[0] = name; + + res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", + 1, params); + PQclear(res); + } + + /* + * send pg_stop_backup asynchronously because we could came + * here from backup_cleanup() after some error caused by + * postgres archive_command problem and in this case we will + * wait for pg_stop_backup() forever. + */ + + if (!pg_stop_backup_is_sent) + { + bool sent = false; + + if (!exclusive_backup) + { + /* + * Stop the non-exclusive backup. Besides stop_lsn it returns from + * pg_stop_backup(false) copy of the backup label and tablespace map + * so they can be written to disk by the caller. + */ + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; + + } + else + { + + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_stop_backup() as lsn"; + } + + sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); + pg_stop_backup_is_sent = true; + if (!sent) + elog(ERROR, "Failed to send pg_stop_backup query"); + } + + /* + * Wait for the result of pg_stop_backup(), + * but no longer than PG_STOP_BACKUP_TIMEOUT seconds + */ + if (pg_stop_backup_is_sent && !in_cleanup) + { + while (1) + { + if (!PQconsumeInput(conn) || PQisBusy(conn)) + { + pg_stop_backup_timeout++; + sleep(1); + + if (interrupted) + { + pgut_cancel(conn); + elog(ERROR, "interrupted during waiting for pg_stop_backup"); + } + + if (pg_stop_backup_timeout == 1) + elog(INFO, "wait for pg_stop_backup()"); + + /* + * If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds, + * send an interrupt. + */ + if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT) + { + pgut_cancel(conn); + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", + PG_STOP_BACKUP_TIMEOUT); + } + } + else + { + res = PQgetResult(conn); + break; + } + } + + /* Check successfull execution of pg_stop_backup() */ + if (!res) + elog(ERROR, "pg_stop backup() failed"); + else + { + switch (PQresultStatus(res)) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + break; + default: + elog(ERROR, "query failed: %s query was: %s", + PQerrorMessage(conn), stop_backup_query); + } + elog(INFO, "pg_stop backup() successfully executed"); + } + + backup_in_progress = false; + + /* Extract timeline and LSN from results of pg_stop_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 2), &xlogid, &xrecoff); + /* Calculate LSN */ + stop_backup_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + if (!XRecOffIsValid(stop_backup_lsn)) + { + stop_backup_lsn = restore_lsn; + } + + if (!XRecOffIsValid(stop_backup_lsn)) + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + + /* Write backup_label and tablespace_map */ + if (!exclusive_backup) + { + Assert(PQnfields(res) >= 4); + pgBackupGetPath(¤t, path, lengthof(path), DATABASE_DIR); + + /* Write backup_label */ + join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); + fp = fopen(backup_label, PG_BINARY_W); + if (fp == NULL) + elog(ERROR, "can't open backup label file \"%s\": %s", + backup_label, strerror(errno)); + + len = strlen(PQgetvalue(res, 0, 3)); + if (fwrite(PQgetvalue(res, 0, 3), 1, len, fp) != len || + fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "can't write backup label file \"%s\": %s", + backup_label, strerror(errno)); + + /* + * It's vital to check if backup_files_list is initialized, + * because we could get here because the backup was interrupted + */ + if (backup_files_list) + { + file = pgFileNew(backup_label, true); + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_BACKUP_LABEL_FILE); + parray_append(backup_files_list, file); + } + } + + if (sscanf(PQgetvalue(res, 0, 0), XID_FMT, &recovery_xid) != 1) + elog(ERROR, + "result of txid_snapshot_xmax() is invalid: %s", + PQgetvalue(res, 0, 0)); + if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) + elog(ERROR, + "result of current_timestamp is invalid: %s", + PQgetvalue(res, 0, 1)); + + /* Get content for tablespace_map from stop_backup results + * in case of non-exclusive backup + */ + if (!exclusive_backup) + val = PQgetvalue(res, 0, 4); + + /* Write tablespace_map */ + if (!exclusive_backup && val && strlen(val) > 0) + { + char tablespace_map[MAXPGPATH]; + + join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); + fp = fopen(tablespace_map, PG_BINARY_W); + if (fp == NULL) + elog(ERROR, "can't open tablespace map file \"%s\": %s", + tablespace_map, strerror(errno)); + + len = strlen(val); + if (fwrite(val, 1, len, fp) != len || + fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "can't write tablespace map file \"%s\": %s", + tablespace_map, strerror(errno)); + + if (backup_files_list) + { + file = pgFileNew(tablespace_map, true); + if (S_ISREG(file->mode)) + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_TABLESPACE_MAP_FILE); + parray_append(backup_files_list, file); + } + } + + if (tablespace_map_content) + PQclear(tablespace_map_content); + PQclear(res); + + if (stream_wal) + { + /* Wait for the completion of stream */ + pthread_join(stream_thread, NULL); + if (stream_thread_arg.ret == 1) + elog(ERROR, "WAL streaming failed"); + } + } + + /* Fill in fields if that is the correct end of backup. */ + if (backup != NULL) + { + char *xlog_path, + stream_xlog_path[MAXPGPATH]; + + /* Wait for stop_lsn to be received by replica */ + if (backup->from_replica) + wait_replica_wal_lsn(stop_backup_lsn, false); + /* + * Wait for stop_lsn to be archived or streamed. + * We wait for stop_lsn in stream mode just in case. + */ + wait_wal_lsn(stop_backup_lsn, false); + + if (stream_wal) + { + pgBackupGetPath2(backup, stream_xlog_path, + lengthof(stream_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); + xlog_path = stream_xlog_path; + } + else + xlog_path = arclog_path; + + backup->tli = get_current_timeline(false); + backup->stop_lsn = stop_backup_lsn; + + elog(LOG, "Getting the Recovery Time from WAL"); + + if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, + backup->start_lsn, backup->stop_lsn, + &backup->recovery_time, &backup->recovery_xid)) + { + backup->recovery_time = recovery_time; + backup->recovery_xid = recovery_xid; + } + } +} + +/* + * Retreive checkpoint_timeout GUC value in seconds. + */ +static int +checkpoint_timeout(void) +{ + PGresult *res; + const char *val; + const char *hintmsg; + int val_int; + + res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); + val = PQgetvalue(res, 0, 0); + + if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + { + PQclear(res); + if (hintmsg) + elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, + hintmsg); + else + elog(ERROR, "Invalid value of checkout_timeout %s", val); + } + + PQclear(res); + + return val_int; +} + +/* + * Notify end of backup to server when "backup_label" is in the root directory + * of the DB cluster. + * Also update backup status to ERROR when the backup is not finished. + */ +static void +backup_cleanup(bool fatal, void *userdata) +{ + /* + * Update status of backup in BACKUP_CONTROL_FILE to ERROR. + * end_time != 0 means backup finished + */ + if (current.status == BACKUP_STATUS_RUNNING && current.end_time == 0) + { + elog(WARNING, "Backup %s is running, setting its status to ERROR", + base36enc(current.start_time)); + current.end_time = time(NULL); + current.status = BACKUP_STATUS_ERROR; + pgBackupWriteBackupControlFile(¤t); + } + + /* + * If backup is in progress, notify stop of backup to PostgreSQL + */ + if (backup_in_progress) + { + elog(WARNING, "backup in progress, stop backup"); + pg_stop_backup(NULL); /* don't care stop_lsn on error case */ + } +} + +/* + * Disconnect backup connection during quit pg_probackup. + */ +static void +backup_disconnect(bool fatal, void *userdata) +{ + pgut_disconnect(backup_conn); + if (master_conn) + pgut_disconnect(master_conn); +} + +/* + * Take a backup of the PGDATA at a file level. + * Copy all directories and files listed in backup_files_list. + * If the file is 'datafile' (regular relation's main fork), read it page by page, + * verify checksum and copy. + * In incremental backup mode, copy only files or datafiles' pages changed after + * previous backup. + */ +static void * +backup_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + + /* backup a file */ + for (i = 0; i < n_backup_files_list; i++) + { + int ret; + struct stat buf; + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + + elog(VERBOSE, "Copying file: \"%s\" ", file->path); + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_backup_files_list, file->path); + + /* stat file to check its current state */ + ret = stat(file->path, &buf); + if (ret == -1) + { + if (errno == ENOENT) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + file->write_size = BYTES_INVALID; + elog(LOG, "File \"%s\" is not found", file->path); + continue; + } + else + { + elog(ERROR, + "can't stat file to backup \"%s\": %s", + file->path, strerror(errno)); + } + } + + /* We have already copied all directories */ + if (S_ISDIR(buf.st_mode)) + continue; + + if (S_ISREG(buf.st_mode)) + { + /* Check that file exist in previous backup */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + char *relative; + pgFile key; + pgFile **prev_file; + + relative = GetRelativePath(file->path, arguments->from_root); + key.path = relative; + + prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, + &key, pgFileComparePath); + if (prev_file) + /* File exists in previous backup */ + file->exists_in_prev = true; + } + /* copy the file into backup */ + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, arguments->to_root, + file->path + strlen(arguments->from_root) + 1); + + /* backup block by block if datafile AND not compressed by cfs*/ + if (!backup_data_file(arguments, to_path, file, + arguments->prev_start_lsn, + current.backup_mode, + compress_alg, compress_level)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); + continue; + } + } + /* TODO: + * Check if file exists in previous backup + * If exists: + * if mtime > start_backup_time of parent backup, + * copy file to backup + * if mtime < start_backup_time + * calculate crc, compare crc to old file + * if crc is the same -> skip file + */ + else if (!copy_file(arguments->from_root, arguments->to_root, file)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); + continue; + } + + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + file->path, file->write_size); + } + else + elog(LOG, "unexpected file type %d", buf.st_mode); + } + + /* Close connection */ + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Extract information about files in backup_list parsing their names: + * - remove temp tables from the list + * - remove unlogged tables from the list (leave the _init fork) + * - set flags for database directories + * - set flags for datafiles + */ +static void +parse_backup_filelist_filenames(parray *files, const char *root) +{ + size_t i = 0; + Oid unlogged_file_reloid = 0; + + while (i < parray_num(files)) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *relative; + int sscanf_result; + + relative = GetRelativePath(file->path, root); + + if (S_ISREG(file->mode) && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative)) + { + /* + * Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + * Legal only in case of 'pg_compression' + */ + if (strcmp(file->name, "pg_compression") == 0) + { + Oid tblspcOid; + Oid dbOid; + char tmp_rel_path[MAXPGPATH]; + /* + * Check that the file is located under + * TABLESPACE_VERSION_DIRECTORY + */ + sscanf_result = sscanf(relative, PG_TBLSPC_DIR "/%u/%s/%u", + &tblspcOid, tmp_rel_path, &dbOid); + + /* Yes, it is */ + if (sscanf_result == 2 && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + set_cfs_datafiles(files, root, relative, i); + } + } + + if (S_ISREG(file->mode) && file->tblspcOid != 0 && + file->name && file->name[0]) + { + if (strcmp(file->forkName, "init") == 0) + { + /* + * Do not backup files of unlogged relations. + * scan filelist backward and exclude these files. + */ + int unlogged_file_num = i - 1; + pgFile *unlogged_file = (pgFile *) parray_get(files, + unlogged_file_num); + + unlogged_file_reloid = file->relOid; + + while (unlogged_file_num >= 0 && + (unlogged_file_reloid != 0) && + (unlogged_file->relOid == unlogged_file_reloid)) + { + pgFileFree(unlogged_file); + parray_remove(files, unlogged_file_num); + + unlogged_file_num--; + i--; + + unlogged_file = (pgFile *) parray_get(files, + unlogged_file_num); + } + } + } + + i++; + } +} + +/* If file is equal to pg_compression, then we consider this tablespace as + * cfs-compressed and should mark every file in this tablespace as cfs-file + * Setting is_cfs is done via going back through 'files' set every file + * that contain cfs_tablespace in his path as 'is_cfs' + * Goings back through array 'files' is valid option possible because of current + * sort rules: + * tblspcOid/TABLESPACE_VERSION_DIRECTORY + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1 + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression + */ +static void +set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) +{ + int len; + int p; + pgFile *prev_file; + char *cfs_tblspc_path; + char *relative_prev_file; + + cfs_tblspc_path = strdup(relative); + if(!cfs_tblspc_path) + elog(ERROR, "Out of memory"); + len = strlen("/pg_compression"); + cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0; + elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); + + for (p = (int) i; p >= 0; p--) + { + prev_file = (pgFile *) parray_get(files, (size_t) p); + relative_prev_file = GetRelativePath(prev_file->path, root); + + elog(VERBOSE, "Checking file in cfs tablespace %s", relative_prev_file); + + if (strstr(relative_prev_file, cfs_tblspc_path) != NULL) + { + if (S_ISREG(prev_file->mode) && prev_file->is_datafile) + { + elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s", + relative_prev_file, prev_file->name); + prev_file->is_cfs = true; + } + } + else + { + elog(VERBOSE, "Breaking on %s", relative_prev_file); + break; + } + } + free(cfs_tblspc_path); +} + +/* + * Find pgfile by given rnode in the backup_files_list + * and add given blkno to its pagemap. + */ +void +process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) +{ + char *path; + char *rel_path; + BlockNumber blkno_inseg; + int segno; + pgFile **file_item; + pgFile f; + + segno = blkno / RELSEG_SIZE; + blkno_inseg = blkno % RELSEG_SIZE; + + rel_path = relpathperm(rnode, forknum); + if (segno > 0) + path = psprintf("%s/%s.%u", pgdata, rel_path, segno); + else + path = psprintf("%s/%s", pgdata, rel_path); + + pg_free(rel_path); + + f.path = path; + /* backup_files_list should be sorted before */ + file_item = (pgFile **) parray_bsearch(backup_files_list, &f, + pgFileComparePath); + + /* + * If we don't have any record of this file in the file map, it means + * that it's a relation that did not have much activity since the last + * backup. We can safely ignore it. If it is a new relation file, the + * backup would simply copy it as-is. + */ + if (file_item) + { + /* We need critical section only we use more than one threads */ + if (num_threads > 1) + pthread_lock(&backup_pagemap_mutex); + + datapagemap_add(&(*file_item)->pagemap, blkno_inseg); + + if (num_threads > 1) + pthread_mutex_unlock(&backup_pagemap_mutex); + } + + pg_free(path); +} + +/* + * Given a list of files in the instance to backup, build a pagemap for each + * data file that has ptrack. Result is saved in the pagemap field of pgFile. + * NOTE we rely on the fact that provided parray is sorted by file->path. + */ +static void +make_pagemap_from_ptrack(parray *files) +{ + size_t i; + Oid dbOid_with_ptrack_init = 0; + Oid tblspcOid_with_ptrack_init = 0; + char *ptrack_nonparsed = NULL; + size_t ptrack_nonparsed_size = 0; + + elog(LOG, "Compiling pagemap"); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + size_t start_addr; + + /* + * If there is a ptrack_init file in the database, + * we must backup all its files, ignoring ptrack files for relations. + */ + if (file->is_database) + { + char *filename = strrchr(file->path, '/'); + + Assert(filename != NULL); + filename++; + + /* + * The function pg_ptrack_get_and_clear_db returns true + * if there was a ptrack_init file. + * Also ignore ptrack files for global tablespace, + * to avoid any possible specific errors. + */ + if ((file->tblspcOid == GLOBALTABLESPACE_OID) || + pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid)) + { + dbOid_with_ptrack_init = file->dbOid; + tblspcOid_with_ptrack_init = file->tblspcOid; + } + } + + if (file->is_datafile) + { + if (file->tblspcOid == tblspcOid_with_ptrack_init && + file->dbOid == dbOid_with_ptrack_init) + { + /* ignore ptrack if ptrack_init exists */ + elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); + file->pagemap_isabsent = true; + continue; + } + + /* get ptrack bitmap once for all segments of the file */ + if (file->segno == 0) + { + /* release previous value */ + pg_free(ptrack_nonparsed); + ptrack_nonparsed_size = 0; + + ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, + file->relOid, &ptrack_nonparsed_size); + } + + if (ptrack_nonparsed != NULL) + { + /* + * pg_ptrack_get_and_clear() returns ptrack with VARHDR cutted out. + * Compute the beginning of the ptrack map related to this segment + * + * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 + * RELSEG_SIZE. Number of Pages per segment: 131072 + * RELSEG_SIZE/HEAPBLOCKS_PER_BYTE. number of bytes in ptrack file needed + * to keep track on one relsegment: 16384 + */ + start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; + + /* + * If file segment was created after we have read ptrack, + * we won't have a bitmap for this segment. + */ + if (start_addr > ptrack_nonparsed_size) + { + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + else + { + + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + { + file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + else + { + file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + + file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); + memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + } + } + else + { + /* + * If ptrack file is missing, try to copy the entire file. + * It can happen in two cases: + * - files were created by commands that bypass buffer manager + * and, correspondingly, ptrack mechanism. + * i.e. CREATE DATABASE + * - target relation was deleted. + */ + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + } + } + elog(LOG, "Pagemap compiled"); +// res = pgut_execute(backup_conn, "SET client_min_messages = warning;", 0, NULL, true); +// PQclear(pgut_execute(backup_conn, "CHECKPOINT;", 0, NULL, true)); +} + + +/* + * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is + * set by pg_stop_backup(). + */ +static bool +stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) +{ + static uint32 prevtimeline = 0; + static XLogRecPtr prevpos = InvalidXLogRecPtr; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + + /* we assume that we get called once at the end of each segment */ + if (segment_finished) + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + + /* + * Note that we report the previous, not current, position here. After a + * timeline switch, xlogpos points to the beginning of the segment because + * that's where we always begin streaming. Reporting the end of previous + * timeline isn't totally accurate, because the next timeline can begin + * slightly before the end of the WAL that we received on the previous + * timeline, but it's close enough for reporting purposes. + */ + if (prevtimeline != 0 && prevtimeline != timeline) + elog(LOG, _("switched to timeline %u at %X/%X\n"), + timeline, (uint32) (prevpos >> 32), (uint32) prevpos); + + if (!XLogRecPtrIsInvalid(stop_backup_lsn)) + { + if (xlogpos > stop_backup_lsn) + { + stop_stream_lsn = xlogpos; + return true; + } + + /* pg_stop_backup() was executed, wait for the completion of stream */ + if (stream_stop_timeout == 0) + { + elog(INFO, "Wait for LSN %X/%X to be streamed", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); + + stream_stop_timeout = checkpoint_timeout(); + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + + stream_stop_begin = time(NULL); + } + + if (time(NULL) - stream_stop_begin > stream_stop_timeout) + elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, + stream_stop_timeout); + } + + prevtimeline = timeline; + prevpos = xlogpos; + + return false; +} + +/* + * Start the log streaming + */ +static void * +StreamLog(void *arg) +{ + XLogRecPtr startpos; + TimeLineID starttli; + StreamThreadArg *stream_arg = (StreamThreadArg *) arg; + + /* + * We must use startpos as start_lsn from start_backup + */ + startpos = current.start_lsn; + starttli = current.tli; + + /* + * Always start streaming at the beginning of a segment + */ + startpos -= startpos % xlog_seg_size; + + /* Initialize timeout */ + stream_stop_timeout = 0; + stream_stop_begin = 0; + + /* + * Start the replication + */ + elog(LOG, _("started streaming WAL at %X/%X (timeline %u)"), + (uint32) (startpos >> 32), (uint32) startpos, starttli); + +#if PG_VERSION_NUM >= 90600 + { + StreamCtl ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + + ctl.startpos = startpos; + ctl.timeline = starttli; + ctl.sysidentifier = NULL; + +#if PG_VERSION_NUM >= 100000 + ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); + ctl.replication_slot = replication_slot; + ctl.stop_socket = PGINVALID_SOCKET; +#else + ctl.basedir = (char *) stream_arg->basedir; +#endif + + ctl.stream_stop = stop_streaming; + ctl.standby_message_timeout = standby_message_timeout; + ctl.partial_suffix = NULL; + ctl.synchronous = false; + ctl.mark_done = false; + + if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) + elog(ERROR, "Problem in receivexlog"); + +#if PG_VERSION_NUM >= 100000 + if (!ctl.walmethod->finish()) + elog(ERROR, "Could not finish writing WAL files: %s", + strerror(errno)); +#endif + } +#else + if(ReceiveXlogStream(stream_arg->conn, startpos, starttli, NULL, + (char *) stream_arg->basedir, stop_streaming, + standby_message_timeout, NULL, false, false) == false) + elog(ERROR, "Problem in receivexlog"); +#endif + + elog(LOG, _("finished streaming WAL at %X/%X (timeline %u)"), + (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, starttli); + stream_arg->ret = 0; + + PQfinish(stream_arg->conn); + stream_arg->conn = NULL; + + return NULL; +} + +/* + * Get lsn of the moment when ptrack was enabled the last time. + */ +static XLogRecPtr +get_last_ptrack_lsn(void) + +{ + PGresult *res; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr lsn; + + res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()", 0, NULL); + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + PQclear(res); + return lsn; +} + +char * +pg_ptrack_get_block(backup_files_arg *arguments, + Oid dbOid, + Oid tblsOid, + Oid relOid, + BlockNumber blknum, + size_t *result_size) +{ + PGresult *res; + char *params[4]; + char *result; + + params[0] = palloc(64); + params[1] = palloc(64); + params[2] = palloc(64); + params[3] = palloc(64); + + /* + * Use tmp_conn, since we may work in parallel threads. + * We can connect to any database. + */ + sprintf(params[0], "%i", tblsOid); + sprintf(params[1], "%i", dbOid); + sprintf(params[2], "%i", relOid); + sprintf(params[3], "%u", blknum); + + if (arguments->backup_conn == NULL) + { + arguments->backup_conn = pgut_connect(pgut_dbname); + } + + if (arguments->cancel_conn == NULL) + arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + + //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, + "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", + 4, (const char **)params, true); + + if (PQnfields(res) != 1) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + if (PQgetisnull(res, 0, 0)) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + + PQclear(res); + + pfree(params[0]); + pfree(params[1]); + pfree(params[2]); + pfree(params[3]); + + return result; +} diff --git a/src/catalog.c b/src/catalog.c new file mode 100644 index 00000000..f3f75277 --- /dev/null +++ b/src/catalog.c @@ -0,0 +1,915 @@ +/*------------------------------------------------------------------------- + * + * catalog.c: backup catalog operation + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; +static pgBackup *readBackupControlFile(const char *path); + +static bool exit_hook_registered = false; +static char lock_file[MAXPGPATH]; + +static void +unlink_lock_atexit(void) +{ + int res; + res = unlink(lock_file); + if (res != 0 && res != ENOENT) + elog(WARNING, "%s: %s", lock_file, strerror(errno)); +} + +/* + * Create a lockfile. + */ +void +catalog_lock(void) +{ + int fd; + char buffer[MAXPGPATH * 2 + 256]; + int ntries; + int len; + int encoded_pid; + pid_t my_pid, + my_p_pid; + + join_path_components(lock_file, backup_instance_path, BACKUP_CATALOG_PID); + + /* + * If the PID in the lockfile is our own PID or our parent's or + * grandparent's PID, then the file must be stale (probably left over from + * a previous system boot cycle). We need to check this because of the + * likelihood that a reboot will assign exactly the same PID as we had in + * the previous reboot, or one that's only one or two counts larger and + * hence the lockfile's PID now refers to an ancestor shell process. We + * allow pg_ctl to pass down its parent shell PID (our grandparent PID) + * via the environment variable PG_GRANDPARENT_PID; this is so that + * launching the postmaster via pg_ctl can be just as reliable as + * launching it directly. There is no provision for detecting + * further-removed ancestor processes, but if the init script is written + * carefully then all but the immediate parent shell will be root-owned + * processes and so the kill test will fail with EPERM. Note that we + * cannot get a false negative this way, because an existing postmaster + * would surely never launch a competing postmaster or pg_ctl process + * directly. + */ + my_pid = getpid(); +#ifndef WIN32 + my_p_pid = getppid(); +#else + + /* + * Windows hasn't got getppid(), but doesn't need it since it's not using + * real kill() either... + */ + my_p_pid = 0; +#endif + + /* + * We need a loop here because of race conditions. But don't loop forever + * (for example, a non-writable $backup_instance_path directory might cause a failure + * that won't go away). 100 tries seems like plenty. + */ + for (ntries = 0;; ntries++) + { + /* + * Try to create the lock file --- O_EXCL makes this atomic. + * + * Think not to make the file protection weaker than 0600. See + * comments below. + */ + fd = open(lock_file, O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd >= 0) + break; /* Success; exit the retry loop */ + + /* + * Couldn't create the pid file. Probably it already exists. + */ + if ((errno != EEXIST && errno != EACCES) || ntries > 100) + elog(ERROR, "could not create lock file \"%s\": %s", + lock_file, strerror(errno)); + + /* + * Read the file to get the old owner's PID. Note race condition + * here: file might have been deleted since we tried to create it. + */ + fd = open(lock_file, O_RDONLY, 0600); + if (fd < 0) + { + if (errno == ENOENT) + continue; /* race condition; try again */ + elog(ERROR, "could not open lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if ((len = read(fd, buffer, sizeof(buffer) - 1)) < 0) + elog(ERROR, "could not read lock file \"%s\": %s", + lock_file, strerror(errno)); + close(fd); + + if (len == 0) + elog(ERROR, "lock file \"%s\" is empty", lock_file); + + buffer[len] = '\0'; + encoded_pid = atoi(buffer); + + if (encoded_pid <= 0) + elog(ERROR, "bogus data in lock file \"%s\": \"%s\"", + lock_file, buffer); + + /* + * Check to see if the other process still exists + * + * Per discussion above, my_pid, my_p_pid can be + * ignored as false matches. + * + * Normally kill() will fail with ESRCH if the given PID doesn't + * exist. + */ + if (encoded_pid != my_pid && encoded_pid != my_p_pid) + { + if (kill(encoded_pid, 0) == 0 || + (errno != ESRCH && errno != EPERM)) + elog(ERROR, "lock file \"%s\" already exists", lock_file); + } + + /* + * Looks like nobody's home. Unlink the file and try again to create + * it. Need a loop because of possible race condition against other + * would-be creators. + */ + if (unlink(lock_file) < 0) + elog(ERROR, "could not remove old lock file \"%s\": %s", + lock_file, strerror(errno)); + } + + /* + * Successfully created the file, now fill it. + */ + snprintf(buffer, sizeof(buffer), "%d\n", my_pid); + + errno = 0; + if (write(fd, buffer, strlen(buffer)) != strlen(buffer)) + { + int save_errno = errno; + + close(fd); + unlink(lock_file); + /* if write didn't set errno, assume problem is no disk space */ + errno = save_errno ? save_errno : ENOSPC; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if (fsync(fd) != 0) + { + int save_errno = errno; + + close(fd); + unlink(lock_file); + errno = save_errno; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if (close(fd) != 0) + { + int save_errno = errno; + + unlink(lock_file); + errno = save_errno; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + + /* + * Arrange to unlink the lock file(s) at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(unlink_lock_atexit); + exit_hook_registered = true; + } +} + +/* + * Read backup meta information from BACKUP_CONTROL_FILE. + * If no backup matches, return NULL. + */ +pgBackup * +read_backup(time_t timestamp) +{ + pgBackup tmp; + char conf_path[MAXPGPATH]; + + tmp.start_time = timestamp; + pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); + + return readBackupControlFile(conf_path); +} + +/* + * Get backup_mode in string representation. + */ +const char * +pgBackupGetBackupMode(pgBackup *backup) +{ + return backupModes[backup->backup_mode]; +} + +static bool +IsDir(const char *dirpath, const char *entry) +{ + char path[MAXPGPATH]; + struct stat st; + + snprintf(path, MAXPGPATH, "%s/%s", dirpath, entry); + + return stat(path, &st) == 0 && S_ISDIR(st.st_mode); +} + +/* + * Create list of backups. + * If 'requested_backup_id' is INVALID_BACKUP_ID, return list of all backups. + * The list is sorted in order of descending start time. + * If valid backup id is passed only matching backup will be added to the list. + */ +parray * +catalog_get_backup_list(time_t requested_backup_id) +{ + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; + parray *backups = NULL; + pgBackup *backup = NULL; + int i; + + /* open backup instance backups directory */ + data_dir = opendir(backup_instance_path); + if (data_dir == NULL) + { + elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, + strerror(errno)); + goto err_proc; + } + + /* scan the directory and list backups */ + backups = parray_new(); + for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) + { + char backup_conf_path[MAXPGPATH]; + char data_path[MAXPGPATH]; + + /* skip not-directory entries and hidden entries */ + if (!IsDir(backup_instance_path, data_ent->d_name) + || data_ent->d_name[0] == '.') + continue; + + /* open subdirectory of specific backup */ + join_path_components(data_path, backup_instance_path, data_ent->d_name); + + /* read backup information from BACKUP_CONTROL_FILE */ + snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); + backup = readBackupControlFile(backup_conf_path); + + /* ignore corrupted backups */ + if (backup) + { + backup->backup_id = backup->start_time; + + if (requested_backup_id != INVALID_BACKUP_ID + && requested_backup_id != backup->start_time) + { + pgBackupFree(backup); + continue; + } + parray_append(backups, backup); + backup = NULL; + } + + if (errno && errno != ENOENT) + { + elog(WARNING, "cannot read data directory \"%s\": %s", + data_ent->d_name, strerror(errno)); + goto err_proc; + } + } + if (errno) + { + elog(WARNING, "cannot read backup root directory \"%s\": %s", + backup_instance_path, strerror(errno)); + goto err_proc; + } + + closedir(data_dir); + data_dir = NULL; + + parray_qsort(backups, pgBackupCompareIdDesc); + + /* Link incremental backups with their ancestors.*/ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *curr = parray_get(backups, i); + + int j; + + if (curr->backup_mode == BACKUP_MODE_FULL) + continue; + + for (j = i+1; j < parray_num(backups); j++) + { + pgBackup *ancestor = parray_get(backups, j); + + if (ancestor->start_time == curr->parent_backup) + { + curr->parent_backup_link = ancestor; + /* elog(INFO, "curr %s, ancestor %s j=%d", base36enc_dup(curr->start_time), + base36enc_dup(ancestor->start_time), j); */ + break; + } + } + } + + return backups; + +err_proc: + if (data_dir) + closedir(data_dir); + if (backup) + pgBackupFree(backup); + if (backups) + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(ERROR, "Failed to get backup list"); + + return NULL; +} + +/* + * Find the last completed backup on given timeline + */ +pgBackup * +catalog_get_last_data_backup(parray *backup_list, TimeLineID tli) +{ + int i; + pgBackup *backup = NULL; + + /* backup_list is sorted in order of descending ID */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->status == BACKUP_STATUS_OK && backup->tli == tli) + return backup; + } + + return NULL; +} + +/* create backup directory in $BACKUP_PATH */ +int +pgBackupCreateDir(pgBackup *backup) +{ + int i; + char path[MAXPGPATH]; + char *subdirs[] = { DATABASE_DIR, NULL }; + + pgBackupGetPath(backup, path, lengthof(path), NULL); + + if (!dir_is_empty(path)) + elog(ERROR, "backup destination is not empty \"%s\"", path); + + dir_create_dir(path, DIR_PERMISSION); + + /* create directories for actual backup files */ + for (i = 0; subdirs[i]; i++) + { + pgBackupGetPath(backup, path, lengthof(path), subdirs[i]); + dir_create_dir(path, DIR_PERMISSION); + } + + return 0; +} + +/* + * Write information about backup.in to stream "out". + */ +void +pgBackupWriteControl(FILE *out, pgBackup *backup) +{ + char timestamp[100]; + + fprintf(out, "#Configuration\n"); + fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); + fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fprintf(out, "compress-alg = %s\n", + deparse_compress_alg(backup->compress_alg)); + fprintf(out, "compress-level = %d\n", backup->compress_level); + fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); + + fprintf(out, "\n#Compatibility\n"); + fprintf(out, "block-size = %u\n", backup->block_size); + fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); + fprintf(out, "checksum-version = %u\n", backup->checksum_version); + fprintf(out, "program-version = %s\n", PROGRAM_VERSION); + if (backup->server_version[0] != '\0') + fprintf(out, "server-version = %s\n", backup->server_version); + + fprintf(out, "\n#Result backup info\n"); + fprintf(out, "timelineid = %d\n", backup->tli); + /* LSN returned by pg_start_backup */ + fprintf(out, "start-lsn = %X/%X\n", + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn); + /* LSN returned by pg_stop_backup */ + fprintf(out, "stop-lsn = %X/%X\n", + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + fprintf(out, "start-time = '%s'\n", timestamp); + if (backup->end_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + fprintf(out, "end-time = '%s'\n", timestamp); + } + fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + fprintf(out, "recovery-time = '%s'\n", timestamp); + } + + /* + * Size of PGDATA directory. The size does not include size of related + * WAL segments in archive 'wal' directory. + */ + if (backup->data_bytes != BYTES_INVALID) + fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + + if (backup->wal_bytes != BYTES_INVALID) + fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + + fprintf(out, "status = %s\n", status2str(backup->status)); + + /* 'parent_backup' is set if it is incremental backup */ + if (backup->parent_backup != 0) + fprintf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); + + /* print connection info except password */ + if (backup->primary_conninfo) + fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); +} + +/* create BACKUP_CONTROL_FILE */ +void +pgBackupWriteBackupControlFile(pgBackup *backup) +{ + FILE *fp = NULL; + char ini_path[MAXPGPATH]; + + pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_CONTROL_FILE); + fp = fopen(ini_path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open configuration file \"%s\": %s", ini_path, + strerror(errno)); + + pgBackupWriteControl(fp, backup); + + fclose(fp); +} + +/* + * Output the list of files to backup catalog DATABASE_FILE_LIST + */ +void +pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) +{ + FILE *fp; + char path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open file list \"%s\": %s", path, + strerror(errno)); + + print_file_list(fp, files, root); + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); +} + +/* + * Read BACKUP_CONTROL_FILE and create pgBackup. + * - Comment starts with ';'. + * - Do not care section. + */ +static pgBackup * +readBackupControlFile(const char *path) +{ + pgBackup *backup = pgut_new(pgBackup); + char *backup_mode = NULL; + char *start_lsn = NULL; + char *stop_lsn = NULL; + char *status = NULL; + char *parent_backup = NULL; + char *program_version = NULL; + char *server_version = NULL; + char *compress_alg = NULL; + int parsed_options; + + pgut_option options[] = + { + {'s', 0, "backup-mode", &backup_mode, SOURCE_FILE_STRICT}, + {'u', 0, "timelineid", &backup->tli, SOURCE_FILE_STRICT}, + {'s', 0, "start-lsn", &start_lsn, SOURCE_FILE_STRICT}, + {'s', 0, "stop-lsn", &stop_lsn, SOURCE_FILE_STRICT}, + {'t', 0, "start-time", &backup->start_time, SOURCE_FILE_STRICT}, + {'t', 0, "end-time", &backup->end_time, SOURCE_FILE_STRICT}, + {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, + {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, + {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, + {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, + {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, + {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, + {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, + {'s', 0, "program-version", &program_version, SOURCE_FILE_STRICT}, + {'s', 0, "server-version", &server_version, SOURCE_FILE_STRICT}, + {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, + {'s', 0, "status", &status, SOURCE_FILE_STRICT}, + {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, + {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, + {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, + {0} + }; + + if (access(path, F_OK) != 0) + { + elog(WARNING, "Control file \"%s\" doesn't exist", path); + pgBackupFree(backup); + return NULL; + } + + pgBackupInit(backup); + parsed_options = pgut_readopt(path, options, WARNING, true); + + if (parsed_options == 0) + { + elog(WARNING, "Control file \"%s\" is empty", path); + pgBackupFree(backup); + return NULL; + } + + if (backup->start_time == 0) + { + elog(WARNING, "Invalid ID/start-time, control file \"%s\" is corrupted", path); + pgBackupFree(backup); + return NULL; + } + + if (backup_mode) + { + backup->backup_mode = parse_backup_mode(backup_mode); + free(backup_mode); + } + + if (start_lsn) + { + uint32 xlogid; + uint32 xrecoff; + + if (sscanf(start_lsn, "%X/%X", &xlogid, &xrecoff) == 2) + backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + elog(WARNING, "Invalid START_LSN \"%s\"", start_lsn); + free(start_lsn); + } + + if (stop_lsn) + { + uint32 xlogid; + uint32 xrecoff; + + if (sscanf(stop_lsn, "%X/%X", &xlogid, &xrecoff) == 2) + backup->stop_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + elog(WARNING, "Invalid STOP_LSN \"%s\"", stop_lsn); + free(stop_lsn); + } + + if (status) + { + if (strcmp(status, "OK") == 0) + backup->status = BACKUP_STATUS_OK; + else if (strcmp(status, "ERROR") == 0) + backup->status = BACKUP_STATUS_ERROR; + else if (strcmp(status, "RUNNING") == 0) + backup->status = BACKUP_STATUS_RUNNING; + else if (strcmp(status, "MERGING") == 0) + backup->status = BACKUP_STATUS_MERGING; + else if (strcmp(status, "DELETING") == 0) + backup->status = BACKUP_STATUS_DELETING; + else if (strcmp(status, "DELETED") == 0) + backup->status = BACKUP_STATUS_DELETED; + else if (strcmp(status, "DONE") == 0) + backup->status = BACKUP_STATUS_DONE; + else if (strcmp(status, "ORPHAN") == 0) + backup->status = BACKUP_STATUS_ORPHAN; + else if (strcmp(status, "CORRUPT") == 0) + backup->status = BACKUP_STATUS_CORRUPT; + else + elog(WARNING, "Invalid STATUS \"%s\"", status); + free(status); + } + + if (parent_backup) + { + backup->parent_backup = base36dec(parent_backup); + free(parent_backup); + } + + if (program_version) + { + StrNCpy(backup->program_version, program_version, + sizeof(backup->program_version)); + pfree(program_version); + } + + if (server_version) + { + StrNCpy(backup->server_version, server_version, + sizeof(backup->server_version)); + pfree(server_version); + } + + if (compress_alg) + backup->compress_alg = parse_compress_alg(compress_alg); + + return backup; +} + +BackupMode +parse_backup_mode(const char *value) +{ + const char *v = value; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0 && pg_strncasecmp("full", v, len) == 0) + return BACKUP_MODE_FULL; + else if (len > 0 && pg_strncasecmp("page", v, len) == 0) + return BACKUP_MODE_DIFF_PAGE; + else if (len > 0 && pg_strncasecmp("ptrack", v, len) == 0) + return BACKUP_MODE_DIFF_PTRACK; + else if (len > 0 && pg_strncasecmp("delta", v, len) == 0) + return BACKUP_MODE_DIFF_DELTA; + + /* Backup mode is invalid, so leave with an error */ + elog(ERROR, "invalid backup-mode \"%s\"", value); + return BACKUP_MODE_INVALID; +} + +const char * +deparse_backup_mode(BackupMode mode) +{ + switch (mode) + { + case BACKUP_MODE_FULL: + return "full"; + case BACKUP_MODE_DIFF_PAGE: + return "page"; + case BACKUP_MODE_DIFF_PTRACK: + return "ptrack"; + case BACKUP_MODE_DIFF_DELTA: + return "delta"; + case BACKUP_MODE_INVALID: + return "invalid"; + } + + return NULL; +} + +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + +/* + * Fill pgBackup struct with default values. + */ +void +pgBackupInit(pgBackup *backup) +{ + backup->backup_id = INVALID_BACKUP_ID; + backup->backup_mode = BACKUP_MODE_INVALID; + backup->status = BACKUP_STATUS_INVALID; + backup->tli = 0; + backup->start_lsn = 0; + backup->stop_lsn = 0; + backup->start_time = (time_t) 0; + backup->end_time = (time_t) 0; + backup->recovery_xid = 0; + backup->recovery_time = (time_t) 0; + + backup->data_bytes = BYTES_INVALID; + backup->wal_bytes = BYTES_INVALID; + + backup->compress_alg = COMPRESS_ALG_DEFAULT; + backup->compress_level = COMPRESS_LEVEL_DEFAULT; + + backup->block_size = BLCKSZ; + backup->wal_block_size = XLOG_BLCKSZ; + backup->checksum_version = 0; + + backup->stream = false; + backup->from_replica = false; + backup->parent_backup = INVALID_BACKUP_ID; + backup->parent_backup_link = NULL; + backup->primary_conninfo = NULL; + backup->program_version[0] = '\0'; + backup->server_version[0] = '\0'; +} + +/* + * Copy backup metadata from **src** into **dst**. + */ +void +pgBackupCopy(pgBackup *dst, pgBackup *src) +{ + pfree(dst->primary_conninfo); + + memcpy(dst, src, sizeof(pgBackup)); + + if (src->primary_conninfo) + dst->primary_conninfo = pstrdup(src->primary_conninfo); +} + +/* free pgBackup object */ +void +pgBackupFree(void *backup) +{ + pgBackup *b = (pgBackup *) backup; + + pfree(b->primary_conninfo); + pfree(backup); +} + +/* Compare two pgBackup with their IDs (start time) in ascending order */ +int +pgBackupCompareId(const void *l, const void *r) +{ + pgBackup *lp = *(pgBackup **)l; + pgBackup *rp = *(pgBackup **)r; + + if (lp->start_time > rp->start_time) + return 1; + else if (lp->start_time < rp->start_time) + return -1; + else + return 0; +} + +/* Compare two pgBackup with their IDs in descending order */ +int +pgBackupCompareIdDesc(const void *l, const void *r) +{ + return -pgBackupCompareId(l, r); +} + +/* + * Construct absolute path of the backup directory. + * If subdir is not NULL, it will be appended after the path. + */ +void +pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) +{ + pgBackupGetPath2(backup, path, len, subdir, NULL); +} + +/* + * Construct absolute path of the backup directory. + * Append "subdir1" and "subdir2" to the backup directory. + */ +void +pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2) +{ + /* If "subdir1" is NULL do not check "subdir2" */ + if (!subdir1) + snprintf(path, len, "%s/%s", backup_instance_path, + base36enc(backup->start_time)); + else if (!subdir2) + snprintf(path, len, "%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1); + /* "subdir1" and "subdir2" is not NULL */ + else + snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1, subdir2); + + make_native_path(path); +} + +/* Find parent base FULL backup for current backup using parent_backup_link, + * return NULL if not found + */ +pgBackup* +find_parent_backup(pgBackup *current_backup) +{ + pgBackup *base_full_backup = NULL; + base_full_backup = current_backup; + + while (base_full_backup->backup_mode != BACKUP_MODE_FULL) + { + /* + * If we haven't found parent for incremental backup, + * mark it and all depending backups as orphaned + */ + if (base_full_backup->parent_backup_link == NULL + || (base_full_backup->status != BACKUP_STATUS_OK + && base_full_backup->status != BACKUP_STATUS_DONE)) + { + pgBackup *orphaned_backup = current_backup; + + while (orphaned_backup != NULL) + { + orphaned_backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(orphaned_backup); + if (base_full_backup->parent_backup_link == NULL) + elog(WARNING, "Backup %s is orphaned because its parent backup is not found", + base36enc(orphaned_backup->start_time)); + else + elog(WARNING, "Backup %s is orphaned because its parent backup is corrupted", + base36enc(orphaned_backup->start_time)); + + orphaned_backup = orphaned_backup->parent_backup_link; + } + + base_full_backup = NULL; + break; + } + + base_full_backup = base_full_backup->parent_backup_link; + } + + return base_full_backup; +} diff --git a/src/configure.c b/src/configure.c new file mode 100644 index 00000000..8b86e438 --- /dev/null +++ b/src/configure.c @@ -0,0 +1,490 @@ +/*------------------------------------------------------------------------- + * + * configure.c: - manage backup catalog. + * + * Copyright (c) 2017-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "utils/logger.h" + +#include "pqexpbuffer.h" + +#include "utils/json.h" + + +static void opt_log_level_console(pgut_option *opt, const char *arg); +static void opt_log_level_file(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); + +static void show_configure_start(void); +static void show_configure_end(void); +static void show_configure(pgBackupConfig *config); + +static void show_configure_json(pgBackupConfig *config); + +static pgBackupConfig *cur_config = NULL; + +static PQExpBufferData show_buf; +static int32 json_level = 0; + +/* + * All this code needs refactoring. + */ + +/* Set configure options */ +int +do_configure(bool show_only) +{ + pgBackupConfig *config = readBackupCatalogConfigFile(); + if (pgdata) + config->pgdata = pgdata; + if (pgut_dbname) + config->pgdatabase = pgut_dbname; + if (host) + config->pghost = host; + if (port) + config->pgport = port; + if (username) + config->pguser = username; + + if (master_host) + config->master_host = master_host; + if (master_port) + config->master_port = master_port; + if (master_db) + config->master_db = master_db; + if (master_user) + config->master_user = master_user; + + if (replica_timeout) + config->replica_timeout = replica_timeout; + + if (archive_timeout) + config->archive_timeout = archive_timeout; + + if (log_level_console) + config->log_level_console = log_level_console; + if (log_level_file) + config->log_level_file = log_level_file; + if (log_filename) + config->log_filename = log_filename; + if (error_log_filename) + config->error_log_filename = error_log_filename; + if (log_directory) + config->log_directory = log_directory; + if (log_rotation_size) + config->log_rotation_size = log_rotation_size; + if (log_rotation_age) + config->log_rotation_age = log_rotation_age; + + if (retention_redundancy) + config->retention_redundancy = retention_redundancy; + if (retention_window) + config->retention_window = retention_window; + + if (compress_alg) + config->compress_alg = compress_alg; + if (compress_level) + config->compress_level = compress_level; + + if (show_only) + show_configure(config); + else + writeBackupCatalogConfigFile(config); + + return 0; +} + +void +pgBackupConfigInit(pgBackupConfig *config) +{ + config->system_identifier = 0; + +#if PG_VERSION_NUM >= 110000 + config->xlog_seg_size = 0; +#else + config->xlog_seg_size = XLOG_SEG_SIZE; +#endif + + config->pgdata = NULL; + config->pgdatabase = NULL; + config->pghost = NULL; + config->pgport = NULL; + config->pguser = NULL; + + config->master_host = NULL; + config->master_port = NULL; + config->master_db = NULL; + config->master_user = NULL; + config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; + + config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; + + config->log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; + config->log_level_file = LOG_LEVEL_FILE_DEFAULT; + config->log_filename = LOG_FILENAME_DEFAULT; + config->error_log_filename = NULL; + config->log_directory = LOG_DIRECTORY_DEFAULT; + config->log_rotation_size = LOG_ROTATION_SIZE_DEFAULT; + config->log_rotation_age = LOG_ROTATION_AGE_DEFAULT; + + config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; + config->retention_window = RETENTION_WINDOW_DEFAULT; + + config->compress_alg = COMPRESS_ALG_DEFAULT; + config->compress_level = COMPRESS_LEVEL_DEFAULT; +} + +void +writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) +{ + uint64 res; + const char *unit; + + fprintf(out, "#Backup instance info\n"); + fprintf(out, "PGDATA = %s\n", config->pgdata); + fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); +#if PG_VERSION_NUM >= 110000 + fprintf(out, "xlog-seg-size = %u\n", config->xlog_seg_size); +#endif + + fprintf(out, "#Connection parameters:\n"); + if (config->pgdatabase) + fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); + if (config->pghost) + fprintf(out, "PGHOST = %s\n", config->pghost); + if (config->pgport) + fprintf(out, "PGPORT = %s\n", config->pgport); + if (config->pguser) + fprintf(out, "PGUSER = %s\n", config->pguser); + + fprintf(out, "#Replica parameters:\n"); + if (config->master_host) + fprintf(out, "master-host = %s\n", config->master_host); + if (config->master_port) + fprintf(out, "master-port = %s\n", config->master_port); + if (config->master_db) + fprintf(out, "master-db = %s\n", config->master_db); + if (config->master_user) + fprintf(out, "master-user = %s\n", config->master_user); + + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Archive parameters:\n"); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "archive-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Logging parameters:\n"); + fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); + fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); + fprintf(out, "log-filename = %s\n", config->log_filename); + if (config->error_log_filename) + fprintf(out, "error-log-filename = %s\n", config->error_log_filename); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + fprintf(out, "log-directory = %s/%s\n", backup_path, config->log_directory); + else + fprintf(out, "log-directory = %s\n", config->log_directory); + /* Convert values from base unit */ + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, (res)?unit:"KB"); + + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, (res)?unit:"min"); + + fprintf(out, "#Retention parameters:\n"); + fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); + fprintf(out, "retention-window = %u\n", config->retention_window); + + fprintf(out, "#Compression parameters:\n"); + + fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); + fprintf(out, "compress-level = %d\n", config->compress_level); +} + +void +writeBackupCatalogConfigFile(pgBackupConfig *config) +{ + char path[MAXPGPATH]; + FILE *fp; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot create %s: %s", + BACKUP_CATALOG_CONF_FILE, strerror(errno)); + + writeBackupCatalogConfig(fp, config); + fclose(fp); +} + + +pgBackupConfig* +readBackupCatalogConfigFile(void) +{ + pgBackupConfig *config = pgut_new(pgBackupConfig); + char path[MAXPGPATH]; + + pgut_option options[] = + { + /* retention options */ + { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, + { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, + /* compression options */ + { 'f', 0, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 0, "compress-level", &(config->compress_level), SOURCE_CMDLINE }, + /* logging options */ + { 'f', 0, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, + { 'f', 0, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, + { 's', 0, "log-filename", &(config->log_filename), SOURCE_CMDLINE }, + { 's', 0, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, + { 's', 0, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, + { 'u', 0, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, + { 'u', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* connection options */ + { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, + { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, + { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, + { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, + { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, + /* replica options */ + { 's', 0, "master-host", &(config->master_host), SOURCE_FILE_STRICT }, + { 's', 0, "master-port", &(config->master_port), SOURCE_FILE_STRICT }, + { 's', 0, "master-db", &(config->master_db), SOURCE_FILE_STRICT }, + { 's', 0, "master-user", &(config->master_user), SOURCE_FILE_STRICT }, + { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* other options */ + { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, +#if PG_VERSION_NUM >= 110000 + {'u', 0, "xlog-seg-size", &config->xlog_seg_size, SOURCE_FILE_STRICT}, +#endif + /* archive options */ + { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + {0} + }; + + cur_config = config; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + + pgBackupConfigInit(config); + pgut_readopt(path, options, ERROR, true); + +#if PG_VERSION_NUM >= 110000 + if (!IsValidWalSegSize(config->xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", config->xlog_seg_size); +#endif + + return config; +} + +/* + * Read xlog-seg-size from BACKUP_CATALOG_CONF_FILE. + */ +uint32 +get_config_xlog_seg_size(void) +{ +#if PG_VERSION_NUM >= 110000 + char path[MAXPGPATH]; + uint32 seg_size; + pgut_option options[] = + { + {'u', 0, "xlog-seg-size", &seg_size, SOURCE_FILE_STRICT}, + {0} + }; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, false); + + if (!IsValidWalSegSize(seg_size)) + elog(ERROR, "Invalid WAL segment size %u", seg_size); + + return seg_size; + +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + +static void +opt_log_level_console(pgut_option *opt, const char *arg) +{ + cur_config->log_level_console = parse_log_level(arg); +} + +static void +opt_log_level_file(pgut_option *opt, const char *arg) +{ + cur_config->log_level_file = parse_log_level(arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + cur_config->compress_alg = parse_compress_alg(arg); +} + +/* + * Initialize configure visualization. + */ +static void +show_configure_start(void) +{ + if (show_format == SHOW_PLAIN) + return; + + /* For now we need buffer only for JSON format */ + json_level = 0; + initPQExpBuffer(&show_buf); +} + +/* + * Finalize configure visualization. + */ +static void +show_configure_end(void) +{ + if (show_format == SHOW_PLAIN) + return; + else + appendPQExpBufferChar(&show_buf, '\n'); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show configure information of pg_probackup. + */ +static void +show_configure(pgBackupConfig *config) +{ + show_configure_start(); + + if (show_format == SHOW_PLAIN) + writeBackupCatalogConfig(stdout, config); + else + show_configure_json(config); + + show_configure_end(); +} + +/* + * Json output. + */ + +static void +show_configure_json(pgBackupConfig *config) +{ + PQExpBuffer buf = &show_buf; + uint64 res; + const char *unit; + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "pgdata", config->pgdata, json_level, false); + + json_add_key(buf, "system-identifier", json_level, true); + appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); + +#if PG_VERSION_NUM >= 110000 + json_add_key(buf, "xlog-seg-size", json_level, true); + appendPQExpBuffer(buf, "%u", config->xlog_seg_size); +#endif + + /* Connection parameters */ + if (config->pgdatabase) + json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); + if (config->pghost) + json_add_value(buf, "pghost", config->pghost, json_level, true); + if (config->pgport) + json_add_value(buf, "pgport", config->pgport, json_level, true); + if (config->pguser) + json_add_value(buf, "pguser", config->pguser, json_level, true); + + /* Replica parameters */ + if (config->master_host) + json_add_value(buf, "master-host", config->master_host, json_level, + true); + if (config->master_port) + json_add_value(buf, "master-port", config->master_port, json_level, + true); + if (config->master_db) + json_add_value(buf, "master-db", config->master_db, json_level, true); + if (config->master_user) + json_add_value(buf, "master-user", config->master_user, json_level, + true); + + json_add_key(buf, "replica-timeout", json_level, true); + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Archive parameters */ + json_add_key(buf, "archive-timeout", json_level, true); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Logging parameters */ + json_add_value(buf, "log-level-console", + deparse_log_level(config->log_level_console), json_level, + true); + json_add_value(buf, "log-level-file", + deparse_log_level(config->log_level_file), json_level, + true); + json_add_value(buf, "log-filename", config->log_filename, json_level, + true); + if (config->error_log_filename) + json_add_value(buf, "error-log-filename", config->error_log_filename, + json_level, true); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + { + char log_directory_fullpath[MAXPGPATH]; + + sprintf(log_directory_fullpath, "%s/%s", + backup_path, config->log_directory); + + json_add_value(buf, "log-directory", log_directory_fullpath, + json_level, true); + } + else + json_add_value(buf, "log-directory", config->log_directory, + json_level, true); + + json_add_key(buf, "log-rotation-size", json_level, true); + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"KB"); + + json_add_key(buf, "log-rotation-age", json_level, true); + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"min"); + + /* Retention parameters */ + json_add_key(buf, "retention-redundancy", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_redundancy); + + json_add_key(buf, "retention-window", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_window); + + /* Compression parameters */ + json_add_value(buf, "compress-algorithm", + deparse_compress_alg(config->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", config->compress_level); + + json_add(buf, JT_END_OBJECT, &json_level); +} diff --git a/src/data.c b/src/data.c new file mode 100644 index 00000000..a66770bc --- /dev/null +++ b/src/data.c @@ -0,0 +1,1407 @@ +/*------------------------------------------------------------------------- + * + * data.c: utils to parse and backup data pages + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "libpq/pqsignal.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum_impl.h" +#include + +#ifdef HAVE_LIBZ +#include +#endif + +#ifdef HAVE_LIBZ +/* Implementation of zlib compression method */ +static int32 +zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, + int level) +{ + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, + level); + + return rc == Z_OK ? compressed_size : rc; +} + +/* Implementation of zlib compression method */ +static int32 +zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) +{ + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); + + return rc == Z_OK ? dest_len : rc; +} +#endif + +/* + * Compresses source into dest using algorithm. Returns the number of bytes + * written in the destination buffer, or -1 if compression fails. + */ +static int32 +do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, int level) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; +#ifdef HAVE_LIBZ + case ZLIB_COMPRESS: + return zlib_compress(dst, dst_size, src, src_size, level); +#endif + case PGLZ_COMPRESS: + return pglz_compress(src, src_size, dst, PGLZ_strategy_always); + } + + return -1; +} + +/* + * Decompresses source into dest using algorithm. Returns the number of bytes + * decompressed in the destination buffer, or -1 if decompression fails. + */ +static int32 +do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; +#ifdef HAVE_LIBZ + case ZLIB_COMPRESS: + return zlib_decompress(dst, dst_size, src, src_size); +#endif + case PGLZ_COMPRESS: + return pglz_decompress(src, src_size, dst, dst_size); + } + + return -1; +} + +/* + * When copying datafiles to backup we validate and compress them block + * by block. Thus special header is required for each data block. + */ +typedef struct BackupPageHeader +{ + BlockNumber block; /* block number */ + int32 compressed_size; +} BackupPageHeader; + +/* Special value for compressed_size field */ +#define PageIsTruncated -2 +#define SkipCurrentPage -3 + +/* Verify page's header */ +static bool +parse_page(Page page, XLogRecPtr *lsn) +{ + PageHeader phdr = (PageHeader) page; + + /* Get lsn from page header */ + *lsn = PageXLogRecPtrGet(phdr->pd_lsn); + + if (PageGetPageSize(phdr) == BLCKSZ && + PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && + phdr->pd_lower >= SizeOfPageHeaderData && + phdr->pd_lower <= phdr->pd_upper && + phdr->pd_upper <= phdr->pd_special && + phdr->pd_special <= BLCKSZ && + phdr->pd_special == MAXALIGN(phdr->pd_special)) + return true; + + return false; +} + +/* Read one page from file directly accessing disk + * return value: + * 0 - if the page is not found + * 1 - if the page is found and valid + * -1 - if the page is found but invalid + */ +static int +read_page_from_file(pgFile *file, BlockNumber blknum, + FILE *in, Page page, XLogRecPtr *page_lsn) +{ + off_t offset = blknum * BLCKSZ; + size_t read_len = 0; + + /* read the block */ + if (fseek(in, offset, SEEK_SET) != 0) + elog(ERROR, "File: %s, could not seek to block %u: %s", + file->path, blknum, strerror(errno)); + + read_len = fread(page, 1, BLCKSZ, in); + + if (read_len != BLCKSZ) + { + /* The block could have been truncated. It is fine. */ + if (read_len == 0) + { + elog(LOG, "File %s, block %u, file was truncated", + file->path, blknum); + return 0; + } + else + elog(WARNING, "File: %s, block %u, expected block size %d," + "but read %lu, try again", + file->path, blknum, BLCKSZ, read_len); + } + + /* + * If we found page with invalid header, at first check if it is zeroed, + * which is a valid state for page. If it is not, read it and check header + * again, because it's possible that we've read a partly flushed page. + * If after several attempts page header is still invalid, throw an error. + * The same idea is applied to checksum verification. + */ + if (!parse_page(page, page_lsn)) + { + int i; + /* Check if the page is zeroed. */ + for(i = 0; i < BLCKSZ && page[i] == 0; i++); + + /* Page is zeroed. No need to check header and checksum. */ + if (i == BLCKSZ) + { + elog(LOG, "File: %s blknum %u, empty page", file->path, blknum); + return 1; + } + + /* + * If page is not completely empty and we couldn't parse it, + * try again several times. If it didn't help, throw error + */ + elog(LOG, "File: %s blknum %u have wrong page header, try again", + file->path, blknum); + return -1; + } + + /* Verify checksum */ + if(current.checksum_version) + { + /* + * If checksum is wrong, sleep a bit and then try again + * several times. If it didn't help, throw error + */ + if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) + != ((PageHeader) page)->pd_checksum) + { + elog(WARNING, "File: %s blknum %u have wrong checksum, try again", + file->path, blknum); + return -1; + } + else + { + /* page header and checksum are correct */ + return 1; + } + } + else + { + /* page header is correct and checksum check is disabled */ + return 1; + } +} + +/* + * Retrieves a page taking the backup mode into account + * and writes it into argument "page". Argument "page" + * should be a pointer to allocated BLCKSZ of bytes. + * + * Prints appropriate warnings/errors/etc into log. + * Returns 0 if page was successfully retrieved + * SkipCurrentPage(-3) if we need to skip this page + * PageIsTruncated(-2) if the page was truncated + */ +static int32 +prepare_page(backup_files_arg *arguments, + pgFile *file, XLogRecPtr prev_backup_start_lsn, + BlockNumber blknum, BlockNumber nblocks, + FILE *in, int *n_skipped, + BackupMode backup_mode, + Page page) +{ + XLogRecPtr page_lsn = 0; + int try_again = 100; + bool page_is_valid = false; + bool page_is_truncated = false; + BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + + /* + * Read the page and verify its header and checksum. + * Under high write load it's possible that we've read partly + * flushed page, so try several times before throwing an error. + */ + if (backup_mode != BACKUP_MODE_DIFF_PTRACK) + { + while(!page_is_valid && try_again) + { + int result = read_page_from_file(file, blknum, + in, page, &page_lsn); + + try_again--; + if (result == 0) + { + /* This block was truncated.*/ + page_is_truncated = true; + /* Page is not actually valid, but it is absent + * and we're not going to reread it or validate */ + page_is_valid = true; + } + + if (result == 1) + page_is_valid = true; + + /* + * If ptrack support is available use it to get invalid block + * instead of rereading it 99 times + */ + //elog(WARNING, "Checksum_Version: %i", current.checksum_version ? 1 : 0); + + if (result == -1 && is_ptrack_support) + { + elog(WARNING, "File %s, block %u, try to fetch via SQL", + file->path, blknum); + break; + } + } + /* + * If page is not valid after 100 attempts to read it + * throw an error. + */ + if(!page_is_valid && !is_ptrack_support) + elog(ERROR, "Data file checksum mismatch. Canceling backup"); + } + + if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) + { + size_t page_size = 0; + Page ptrack_page = NULL; + ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, + file->relOid, absolute_blknum, &page_size); + + if (ptrack_page == NULL) + { + /* This block was truncated.*/ + page_is_truncated = true; + } + else if (page_size != BLCKSZ) + { + free(ptrack_page); + elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", + file->path, absolute_blknum, BLCKSZ, page_size); + } + else + { + /* + * We need to copy the page that was successfully + * retreieved from ptrack into our output "page" parameter. + * We must set checksum here, because it is outdated + * in the block recieved from shared buffers. + */ + memcpy(page, ptrack_page, BLCKSZ); + free(ptrack_page); + if (is_checksum_enabled) + ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); + } + /* get lsn from page, provided by pg_ptrack_get_block() */ + if (backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev && + !page_is_truncated && + !parse_page(page, &page_lsn)) + elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " + "Possible risk of a memory corruption"); + + } + + if (backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev && + !page_is_truncated && + page_lsn < prev_backup_start_lsn) + { + elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path); + (*n_skipped)++; + return SkipCurrentPage; + } + + if (page_is_truncated) + return PageIsTruncated; + + return 0; +} + +static void +compress_and_backup_page(pgFile *file, BlockNumber blknum, + FILE *in, FILE *out, pg_crc32 *crc, + int page_state, Page page, + CompressAlg calg, int clevel) +{ + BackupPageHeader header; + size_t write_buffer_size = sizeof(header); + char write_buffer[BLCKSZ+sizeof(header)]; + char compressed_page[BLCKSZ]; + + if(page_state == SkipCurrentPage) + return; + + header.block = blknum; + header.compressed_size = page_state; + + if(page_state == PageIsTruncated) + { + /* + * The page was truncated. Write only header + * to know that we must truncate restored file + */ + memcpy(write_buffer, &header, sizeof(header)); + } + else + { + /* The page was not truncated, so we need to compress it */ + header.compressed_size = do_compress(compressed_page, BLCKSZ, + page, BLCKSZ, calg, clevel); + + file->compress_alg = calg; + file->read_size += BLCKSZ; + Assert (header.compressed_size <= BLCKSZ); + + /* The page was successfully compressed. */ + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), + compressed_page, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + /* Nonpositive value means that compression failed. Write it as is. */ + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page, BLCKSZ); + write_buffer_size += header.compressed_size; + } + } + + /* elog(VERBOSE, "backup blkno %u, compressed_size %d write_buffer_size %ld", + blknum, header.compressed_size, write_buffer_size); */ + + /* Update CRC */ + COMP_CRC32C(*crc, write_buffer, write_buffer_size); + + /* write data page */ + if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) + { + int errno_tmp = errno; + + fclose(in); + fclose(out); + elog(ERROR, "File: %s, cannot write backup at block %u : %s", + file->path, blknum, strerror(errno_tmp)); + } + + file->write_size += write_buffer_size; +} + +/* + * Backup data file in the from_root directory to the to_root directory with + * same relative path. If prev_backup_start_lsn is not NULL, only pages with + * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental backup), validate checksum, optionally compress and write to + * backup with special header. + */ +bool +backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel) +{ + FILE *in; + FILE *out; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int n_blocks_read = 0; + int page_state; + char curr_page[BLCKSZ]; + + /* + * Skip unchanged file only if it exists in previous backup. + * This way we can correctly handle null-sized files which are + * not tracked by pagemap and thus always marked as unchanged. + */ + if ((backup_mode == BACKUP_MODE_DIFF_PAGE || + backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->pagemap.bitmapsize == PageBitmapIsEmpty && + file->exists_in_prev && !file->pagemap_isabsent) + { + /* + * There are no changed blocks since last backup. We want make + * incremental backup, so we should exit. + */ + elog(VERBOSE, "Skipping the unchanged file: %s", file->path); + return false; + } + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + INIT_CRC32C(file->crc); + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(file->crc); + + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + { + elog(LOG, "File \"%s\" is not found", file->path); + return false; + } + + elog(ERROR, "cannot open file \"%s\": %s", + file->path, strerror(errno)); + } + + if (file->size % BLCKSZ != 0) + { + fclose(in); + elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size); + } + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + nblocks = file->size/BLCKSZ; + + /* open backup file for write */ + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open backup file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* + * Read each page, verify checksum and write it to backup. + * If page map is empty or file is not present in previous backup + * backup all pages of the relation. + * + * We will enter here if backup_mode is FULL or DELTA. + */ + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev) + { + for (blknum = 0; blknum < nblocks; blknum++) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } + if (backup_mode == BACKUP_MODE_DIFF_DELTA) + file->n_blocks = n_blocks_read; + } + /* + * If page map is not empty we scan only changed blocks. + * + * We will enter here if backup_mode is PAGE or PTRACK. + */ + else + { + datapagemap_iterator_t *iter; + iter = datapagemap_iterate(&file->pagemap); + while (datapagemap_next(iter, &blknum)) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } + + pg_free(file->pagemap.bitmap); + pg_free(iter); + } + + /* update file permission */ + if (chmod(to_path, FILE_PERMISSION) == -1) + { + int errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", file->path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write backup file \"%s\": %s", + to_path, strerror(errno)); + fclose(in); + + FIN_CRC32C(file->crc); + + /* + * If we have pagemap then file in the backup can't be a zero size. + * Otherwise, we will clear the last file. + */ + if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) + { + if (remove(to_path) == -1) + elog(ERROR, "cannot remove file \"%s\": %s", to_path, + strerror(errno)); + return false; + } + + return true; +} + +/* + * Restore files in the from_root directory to the to_root directory with + * same relative path. + * + * If write_header is true then we add header to each restored block, currently + * it is used for MERGE command. + */ +void +restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, + bool write_header) +{ + FILE *in = NULL; + FILE *out = NULL; + BackupPageHeader header; + BlockNumber blknum = 0, + truncate_from = 0; + bool need_truncate = false; + + /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ + if (file->write_size != BYTES_INVALID) + { + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + elog(ERROR, "cannot open backup file \"%s\": %s", file->path, + strerror(errno)); + } + } + + /* + * Open backup file for write. We use "r+" at first to overwrite only + * modified pages for differential restore. If the file does not exist, + * re-open it with "w" to create an empty file. + */ + out = fopen(to_path, PG_BINARY_R "+"); + if (out == NULL && errno == ENOENT) + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open restore target file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + while (true) + { + off_t write_pos; + size_t read_len; + DataPage compressed_page; /* used as read buffer */ + DataPage page; + + /* File didn`t changed. Nothig to copy */ + if (file->write_size == BYTES_INVALID) + break; + + /* + * We need to truncate result file if data file in a incremental backup + * less than data file in a full backup. We know it thanks to n_blocks. + * + * It may be equal to -1, then we don't want to truncate the result + * file. + */ + if (file->n_blocks != BLOCKNUM_INVALID && + (blknum + 1) > file->n_blocks) + { + truncate_from = blknum; + need_truncate = true; + break; + } + + /* read BackupPageHeader */ + read_len = fread(&header, 1, sizeof(header), in); + if (read_len != sizeof(header)) + { + int errno_tmp = errno; + if (read_len == 0 && feof(in)) + break; /* EOF found */ + else if (read_len != 0 && feof(in)) + elog(ERROR, + "odd size page found at block %u of \"%s\"", + blknum, file->path); + else + elog(ERROR, "cannot read header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno_tmp)); + } + + if (header.block < blknum) + elog(ERROR, "backup is broken at file->path %s block %u", + file->path, blknum); + + blknum = header.block; + + if (header.compressed_size == PageIsTruncated) + { + /* + * Backup contains information that this block was truncated. + * We need to truncate file to this length. + */ + truncate_from = blknum; + need_truncate = true; + break; + } + + Assert(header.compressed_size <= BLCKSZ); + + read_len = fread(compressed_page.data, 1, + MAXALIGN(header.compressed_size), in); + if (read_len != MAXALIGN(header.compressed_size)) + elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + blknum, file->path, read_len, header.compressed_size); + + if (header.compressed_size != BLCKSZ) + { + int32 uncompressed_size = 0; + + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + MAXALIGN(header.compressed_size), + file->compress_alg); + + if (uncompressed_size != BLCKSZ) + elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + file->path, uncompressed_size); + } + + write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : + blknum * BLCKSZ; + + /* + * Seek and write the restored page. + */ + if (fseek(out, write_pos, SEEK_SET) < 0) + elog(ERROR, "cannot seek block %u of \"%s\": %s", + blknum, to_path, strerror(errno)); + + if (write_header) + { + if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) + elog(ERROR, "cannot write header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + + if (header.compressed_size < BLCKSZ) + { + if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + else + { + /* if page wasn't compressed, we've read full block */ + if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + } + + /* + * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do + * But during DELTA backup we read every file in PGDATA and thus DELTA backup + * knows exact size of every file at the time of backup. + * So when restoring file from DELTA backup we, knowning it`s size at + * a time of a backup, can truncate file to this size. + */ + if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) + { + size_t file_size = 0; + + /* get file current size */ + fseek(out, 0, SEEK_END); + file_size = ftell(out); + + if (file_size > file->n_blocks * BLCKSZ) + { + truncate_from = file->n_blocks; + need_truncate = true; + } + } + + if (need_truncate) + { + off_t write_pos; + + write_pos = (write_header) ? truncate_from * (BLCKSZ + sizeof(header)) : + truncate_from * BLCKSZ; + + /* + * Truncate file to this length. + */ + if (ftruncate(fileno(out), write_pos) != 0) + elog(ERROR, "cannot truncate \"%s\": %s", + file->path, strerror(errno)); + elog(INFO, "Delta truncate file %s to block %u", + file->path, truncate_from); + } + + /* update file permission */ + if (chmod(to_path, file->mode) == -1) + { + int errno_tmp = errno; + + if (in) + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + if (in) + fclose(in); +} + +/* + * Copy file to backup. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +bool +copy_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + struct stat st; + pg_crc32 crc; + + INIT_CRC32C(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(crc); + file->crc = crc; + + /* maybe deleted, it's not error */ + if (errno == ENOENT) + return false; + + elog(ERROR, "cannot open source file \"%s\": %s", file->path, + strerror(errno)); + } + + /* open backup file for write */ + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot stat \"%s\": %s", file->path, + strerror(errno)); + } + + /* copy content and calc CRC */ + for (;;) + { + read_len = 0; + + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + break; + + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->read_size += read_len; + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* copy odd part. */ + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->read_size += read_len; + } + + file->write_size = (int64) file->read_size; + /* finish CRC calculation and store into pgFile */ + FIN_CRC32C(crc); + file->crc = crc; + + /* update file permission */ + if (chmod(to_path, st.st_mode) == -1) + { + errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + fclose(in); + + return true; +} + +/* + * Move file from one backup to another. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +void +move_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + if (rename(file->path, to_path) == -1) + elog(ERROR, "Cannot move file \"%s\" to path \"%s\": %s", + file->path, to_path, strerror(errno)); +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf, int errnum) +{ + int gz_errnum; + const char *errmsg; + + errmsg = gzerror(gzf, &gz_errnum); + if (gz_errnum == Z_ERRNO) + return strerror(errnum); + else + return errmsg; +} +#endif + +/* + * Copy file attributes + */ +static void +copy_meta(const char *from_path, const char *to_path, bool unlink_on_error) +{ + struct stat st; + + if (stat(from_path, &st) == -1) + { + if (unlink_on_error) + unlink(to_path); + elog(ERROR, "Cannot stat file \"%s\": %s", + from_path, strerror(errno)); + } + + if (chmod(to_path, st.st_mode) == -1) + { + if (unlink_on_error) + unlink(to_path); + elog(ERROR, "Cannot change mode of file \"%s\": %s", + to_path, strerror(errno)); + } +} + +/* + * Copy WAL segment from pgdata to archive catalog with possible compression. + */ +void +push_wal_file(const char *from_path, const char *to_path, bool is_compress, + bool overwrite) +{ + FILE *in = NULL; + FILE *out=NULL; + char buf[XLOG_BLCKSZ]; + const char *to_path_p = to_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + +#ifdef HAVE_LIBZ + char gz_to_path[MAXPGPATH]; + gzFile gz_out = NULL; +#endif + + /* open file for read */ + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, + strerror(errno)); + + /* open backup file for write */ +#ifdef HAVE_LIBZ + if (is_compress) + { + snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); + + if (!overwrite && fileExists(gz_to_path)) + elog(ERROR, "WAL segment \"%s\" already exists.", gz_to_path); + + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); + + gz_out = gzopen(to_path_temp, PG_BINARY_W); + if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) + elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", + compress_level, to_path_temp, get_gz_error(gz_out, errno)); + + to_path_p = gz_to_path; + } + else +#endif + { + if (!overwrite && fileExists(to_path)) + elog(ERROR, "WAL segment \"%s\" already exists.", to_path); + + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fopen(to_path_temp, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + + /* copy content */ + for (;;) + { + size_t read_len = 0; + + read_len = fread(buf, 1, sizeof(buf), in); + + if (ferror(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, + "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + if (read_len > 0) + { +#ifdef HAVE_LIBZ + if (is_compress) + { + if (gzwrite(gz_out, buf, read_len) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + } + + if (feof(in) || read_len == 0) + break; + } + +#ifdef HAVE_LIBZ + if (is_compress) + { + if (gzclose(gz_out) != 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + + if (fclose(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + /* update file permission. */ + copy_meta(from_path, to_path_temp, true); + + if (rename(to_path_temp, to_path_p) < 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path_p, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_compress) + elog(INFO, "WAL file compressed to \"%s\"", gz_to_path); +#endif +} + +/* + * Copy WAL segment from archive catalog to pgdata with possible decompression. + */ +void +get_wal_file(const char *from_path, const char *to_path) +{ + FILE *in = NULL; + FILE *out; + char buf[XLOG_BLCKSZ]; + const char *from_path_p = from_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + bool is_decompress = false; + +#ifdef HAVE_LIBZ + char gz_from_path[MAXPGPATH]; + gzFile gz_in = NULL; +#endif + + /* open file for read */ + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) + { +#ifdef HAVE_LIBZ + /* + * Maybe we need to decompress the file. Check it with .gz + * extension. + */ + snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); + gz_in = gzopen(gz_from_path, PG_BINARY_R); + if (gz_in == NULL) + { + if (errno == ENOENT) + { + /* There is no compressed file too, raise an error below */ + } + /* Cannot open compressed file for some reason */ + else + elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", + gz_from_path, strerror(errno)); + } + else + { + /* Found compressed file */ + is_decompress = true; + from_path_p = gz_from_path; + } +#endif + /* Didn't find compressed file */ + if (!is_decompress) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", + from_path, strerror(errno)); + } + + /* open backup file for write */ + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fopen(to_path_temp, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + + /* copy content */ + for (;;) + { + size_t read_len = 0; + +#ifdef HAVE_LIBZ + if (is_decompress) + { + read_len = gzread(gz_in, buf, sizeof(buf)); + if (read_len != sizeof(buf) && !gzeof(gz_in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot read compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + read_len = fread(buf, 1, sizeof(buf), in); + if (ferror(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, + strerror(errno_temp)); + } + } + + /* Check for EOF */ +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (gzeof(gz_in) || read_len == 0) + break; + } + else +#endif + { + if (feof(in) || read_len == 0) + break; + } + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (gzclose(gz_in) != 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + if (fclose(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + /* update file permission. */ + copy_meta(from_path_p, to_path_temp, true); + + if (rename(to_path_temp, to_path) < 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + elog(INFO, "WAL file decompressed from \"%s\"", gz_from_path); +#endif +} + +/* + * Calculate checksum of various files which are not copied from PGDATA, + * but created in process of backup, such as stream XLOG files, + * PG_TABLESPACE_MAP_FILE and PG_BACKUP_LABEL_FILE. + */ +bool +calc_file_checksum(pgFile *file) +{ + FILE *in; + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + struct stat st; + pg_crc32 crc; + + Assert(S_ISREG(file->mode)); + INIT_CRC32C(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(crc); + file->crc = crc; + + /* maybe deleted, it's not error */ + if (errno == ENOENT) + return false; + + elog(ERROR, "cannot open source file \"%s\": %s", file->path, + strerror(errno)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + elog(ERROR, "cannot stat \"%s\": %s", file->path, + strerror(errno)); + } + + for (;;) + { + read_len = fread(buf, 1, sizeof(buf), in); + + if(read_len == 0) + break; + + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->write_size += read_len; + file->read_size += read_len; + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* finish CRC calculation and store into pgFile */ + FIN_CRC32C(crc); + file->crc = crc; + + fclose(in); + + return true; +} diff --git a/src/delete.c b/src/delete.c new file mode 100644 index 00000000..de29d2cf --- /dev/null +++ b/src/delete.c @@ -0,0 +1,464 @@ +/*------------------------------------------------------------------------- + * + * delete.c: delete backup files. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +static int pgBackupDeleteFiles(pgBackup *backup); +static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size); + +int +do_delete(time_t backup_id) +{ + int i; + parray *backup_list, + *delete_list; + pgBackup *target_backup = NULL; + time_t parent_id = 0; + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get complete list of backups */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + if (backup_id != 0) + { + delete_list = parray_new(); + + /* Find backup to be deleted and make increment backups array to be deleted */ + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->start_time == backup_id) + { + parray_append(delete_list, backup); + + /* + * Do not remove next backups, if target backup was finished + * incorrectly. + */ + if (backup->status == BACKUP_STATUS_ERROR) + break; + + /* Save backup id to retreive increment backups */ + parent_id = backup->start_time; + target_backup = backup; + } + else if (target_backup) + { + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->parent_backup == parent_id) + { + /* Append to delete list increment backup */ + parray_append(delete_list, backup); + /* Save backup id to retreive increment backups */ + parent_id = backup->start_time; + } + else + break; + } + } + + if (parray_num(delete_list) == 0) + elog(ERROR, "no backup found, cannot delete"); + + /* Delete backups from the end of list */ + for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); + + if (interrupted) + elog(ERROR, "interrupted during delete backup"); + + pgBackupDeleteFiles(backup); + } + + parray_free(delete_list); + } + + /* Clean WAL segments */ + if (delete_wal) + { + Assert(target_backup); + + /* Find oldest LSN, used by backups */ + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->status == BACKUP_STATUS_OK) + { + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + break; + } + } + + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + } + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + return 0; +} + +/* + * Remove backups by retention policy. Retention policy is configured by + * retention_redundancy and retention_window variables. + */ +int +do_retention_purge(void) +{ + parray *backup_list; + uint32 backup_num; + size_t i; + time_t days_threshold = time(NULL) - (retention_window * 60 * 60 * 24); + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + bool keep_next_backup = true; /* Do not delete first full backup */ + bool backup_deleted = false; /* At least one backup was deleted */ + + if (delete_expired) + { + if (retention_redundancy > 0) + elog(LOG, "REDUNDANCY=%u", retention_redundancy); + if (retention_window > 0) + elog(LOG, "WINDOW=%u", retention_window); + + if (retention_redundancy == 0 + && retention_window == 0) + { + elog(WARNING, "Retention policy is not set"); + if (!delete_wal) + return 0; + } + } + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get a complete list of backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + if (parray_num(backup_list) == 0) + { + elog(INFO, "backup list is empty, purging won't be executed"); + return 0; + } + + /* Find target backups to be deleted */ + if (delete_expired && + (retention_redundancy > 0 || retention_window > 0)) + { + backup_num = 0; + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + uint32 backup_num_evaluate = backup_num; + + /* Consider only validated and correct backups */ + if (backup->status != BACKUP_STATUS_OK) + continue; + /* + * When a valid full backup was found, we can delete the + * backup that is older than it using the number of generations. + */ + if (backup->backup_mode == BACKUP_MODE_FULL) + backup_num++; + + /* Evaluate retention_redundancy if this backup is eligible for removal */ + if (keep_next_backup || + retention_redundancy >= backup_num_evaluate + 1 || + (retention_window > 0 && backup->recovery_time >= days_threshold)) + { + /* Save LSN and Timeline to remove unnecessary WAL segments */ + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + + /* Save parent backup of this incremental backup */ + if (backup->backup_mode != BACKUP_MODE_FULL) + keep_next_backup = true; + /* + * Previous incremental backup was kept or this is first backup + * so do not delete this backup. + */ + else + keep_next_backup = false; + + continue; + } + + /* Delete backup and update status to DELETED */ + pgBackupDeleteFiles(backup); + backup_deleted = true; + } + } + + /* + * If oldest_lsn and oldest_tli weren`t set because previous step was skipped + * then set them now if we are going to purge WAL + */ + if (delete_wal && (XLogRecPtrIsInvalid(oldest_lsn))) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) - 1); + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + } + + /* Be paranoid */ + if (XLogRecPtrIsInvalid(oldest_lsn)) + elog(ERROR, "Not going to purge WAL because LSN is invalid"); + + /* Purge WAL files */ + if (delete_wal) + { + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + if (backup_deleted) + elog(INFO, "Purging finished"); + else + elog(INFO, "Nothing to delete by retention policy"); + + return 0; +} + +/* + * Delete backup files of the backup and update the status of the backup to + * BACKUP_STATUS_DELETED. + */ +static int +pgBackupDeleteFiles(pgBackup *backup) +{ + size_t i; + char path[MAXPGPATH]; + char timestamp[100]; + parray *files; + + /* + * If the backup was deleted already, there is nothing to do. + */ + if (backup->status == BACKUP_STATUS_DELETED) + return 0; + + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + + elog(INFO, "delete: %s %s", + base36enc(backup->start_time), timestamp); + + /* + * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which + * the error occurs before deleting all backup files. + */ + backup->status = BACKUP_STATUS_DELETING; + pgBackupWriteBackupControlFile(backup); + + /* list files to be deleted */ + files = parray_new(); + pgBackupGetPath(backup, path, lengthof(path), NULL); + dir_list_file(files, path, false, true, true); + + /* delete leaf node first */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + /* print progress */ + elog(VERBOSE, "delete file(%zd/%lu) \"%s\"", i + 1, + (unsigned long) parray_num(files), file->path); + + if (remove(file->path)) + { + elog(WARNING, "can't remove \"%s\": %s", file->path, + strerror(errno)); + parray_walk(files, pgFileFree); + parray_free(files); + + return 1; + } + } + + parray_walk(files, pgFileFree); + parray_free(files); + backup->status = BACKUP_STATUS_DELETED; + + return 0; +} + +/* + * Deletes WAL segments up to oldest_lsn or all WAL segments (if all backups + * was deleted and so oldest_lsn is invalid). + * + * oldest_lsn - if valid, function deletes WAL segments, which contain lsn + * older than oldest_lsn. If it is invalid function deletes all WAL segments. + * oldest_tli - is used to construct oldest WAL segment in addition to + * oldest_lsn. + */ +static void +delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size) +{ + XLogSegNo targetSegNo; + char oldestSegmentNeeded[MAXFNAMELEN]; + DIR *arcdir; + struct dirent *arcde; + char wal_file[MAXPGPATH]; + char max_wal_file[MAXPGPATH]; + char min_wal_file[MAXPGPATH]; + int rc; + + max_wal_file[0] = '\0'; + min_wal_file[0] = '\0'; + + if (!XLogRecPtrIsInvalid(oldest_lsn)) + { + GetXLogSegNo(oldest_lsn, targetSegNo, xlog_seg_size); + GetXLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo, + xlog_seg_size); + + elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded); + } + else + elog(LOG, "removing all WAL segments"); + + /* + * Now it is time to do the actual work and to remove all the segments + * not needed anymore. + */ + if ((arcdir = opendir(arclog_path)) != NULL) + { + while (errno = 0, (arcde = readdir(arcdir)) != NULL) + { + /* + * We ignore the timeline part of the WAL segment identifiers in + * deciding whether a segment is still needed. This ensures that + * we won't prematurely remove a segment from a parent timeline. + * We could probably be a little more proactive about removing + * segments of non-parent timelines, but that would be a whole lot + * more complicated. + * + * We use the alphanumeric sorting property of the filenames to + * decide which ones are earlier than the exclusiveCleanupFileName + * file. Note that this means files are not removed in the order + * they were originally written, in case this worries you. + * + * We also should not forget that WAL segment can be compressed. + */ + if (IsXLogFileName(arcde->d_name) || + IsPartialXLogFileName(arcde->d_name) || + IsBackupHistoryFileName(arcde->d_name) || + IsCompressedXLogFileName(arcde->d_name)) + { + if (XLogRecPtrIsInvalid(oldest_lsn) || + strncmp(arcde->d_name + 8, oldestSegmentNeeded + 8, 16) < 0) + { + /* + * Use the original file name again now, including any + * extension that might have been chopped off before testing + * the sequence. + */ + snprintf(wal_file, MAXPGPATH, "%s/%s", + arclog_path, arcde->d_name); + + rc = unlink(wal_file); + if (rc != 0) + { + elog(WARNING, "could not remove file \"%s\": %s", + wal_file, strerror(errno)); + break; + } + elog(LOG, "removed WAL segment \"%s\"", wal_file); + + if (max_wal_file[0] == '\0' || + strcmp(max_wal_file + 8, arcde->d_name + 8) < 0) + strcpy(max_wal_file, arcde->d_name); + + if (min_wal_file[0] == '\0' || + strcmp(min_wal_file + 8, arcde->d_name + 8) > 0) + strcpy(min_wal_file, arcde->d_name); + } + } + } + + if (min_wal_file[0] != '\0') + elog(INFO, "removed min WAL segment \"%s\"", min_wal_file); + if (max_wal_file[0] != '\0') + elog(INFO, "removed max WAL segment \"%s\"", max_wal_file); + + if (errno) + elog(WARNING, "could not read archive location \"%s\": %s", + arclog_path, strerror(errno)); + if (closedir(arcdir)) + elog(WARNING, "could not close archive location \"%s\": %s", + arclog_path, strerror(errno)); + } + else + elog(WARNING, "could not open archive location \"%s\": %s", + arclog_path, strerror(errno)); +} + + +/* Delete all backup files and wal files of given instance. */ +int +do_delete_instance(void) +{ + parray *backup_list; + int i; + char instance_config_path[MAXPGPATH]; + + /* Delete all backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + pgBackupDeleteFiles(backup); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + /* Delete all wal files. */ + delete_walfiles(InvalidXLogRecPtr, 0, xlog_seg_size); + + /* Delete backup instance config file */ + join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + if (remove(instance_config_path)) + { + elog(ERROR, "can't remove \"%s\": %s", instance_config_path, + strerror(errno)); + } + + /* Delete instance root directories */ + if (rmdir(backup_instance_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + if (rmdir(arclog_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + + elog(INFO, "Instance '%s' successfully deleted", instance_name); + return 0; +} diff --git a/src/dir.c b/src/dir.c new file mode 100644 index 00000000..a08bd934 --- /dev/null +++ b/src/dir.c @@ -0,0 +1,1491 @@ +/*------------------------------------------------------------------------- + * + * dir.c: directory operation utility. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include + +#include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" +#include "datapagemap.h" + +/* + * The contents of these directories are removed or recreated during server + * start so they are not included in backups. The directories themselves are + * kept and included as empty to preserve access permissions. + */ +const char *pgdata_exclude_dir[] = +{ + PG_XLOG_DIR, + /* + * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even + * when stats_temp_directory is set because PGSS_TEXT_FILE is always created + * there. + */ + "pg_stat_tmp", + "pgsql_tmp", + + /* + * It is generally not useful to backup the contents of this directory even + * if the intention is to restore to another master. See backup.sgml for a + * more detailed description. + */ + "pg_replslot", + + /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ + "pg_dynshmem", + + /* Contents removed on startup, see AsyncShmemInit(). */ + "pg_notify", + + /* + * Old contents are loaded for possible debugging but are not required for + * normal operation, see OldSerXidInit(). + */ + "pg_serial", + + /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */ + "pg_snapshots", + + /* Contents zeroed on startup, see StartupSUBTRANS(). */ + "pg_subtrans", + + /* end of list */ + NULL, /* pg_log will be set later */ + NULL +}; + +static char *pgdata_exclude_files[] = +{ + /* Skip auto conf temporary file. */ + "postgresql.auto.conf.tmp", + + /* Skip current log file temporary file */ + "current_logfiles.tmp", + "recovery.conf", + "postmaster.pid", + "postmaster.opts", + NULL +}; + +static char *pgdata_exclude_files_non_exclusive[] = +{ + /*skip in non-exclusive backup */ + "backup_label", + "tablespace_map", + NULL +}; + +/* Tablespace mapping structures */ + +typedef struct TablespaceListCell +{ + struct TablespaceListCell *next; + char old_dir[MAXPGPATH]; + char new_dir[MAXPGPATH]; +} TablespaceListCell; + +typedef struct TablespaceList +{ + TablespaceListCell *head; + TablespaceListCell *tail; +} TablespaceList; + +typedef struct TablespaceCreatedListCell +{ + struct TablespaceCreatedListCell *next; + char link_name[MAXPGPATH]; + char linked_dir[MAXPGPATH]; +} TablespaceCreatedListCell; + +typedef struct TablespaceCreatedList +{ + TablespaceCreatedListCell *head; + TablespaceCreatedListCell *tail; +} TablespaceCreatedList; + +static int BlackListCompare(const void *str1, const void *str2); + +static bool dir_check_file(const char *root, pgFile *file); +static void dir_list_file_internal(parray *files, const char *root, + pgFile *parent, bool exclude, + bool omit_symlink, parray *black_list); + +static void list_data_directories(parray *files, const char *path, bool is_root, + bool exclude); + +/* Tablespace mapping */ +static TablespaceList tablespace_dirs = {NULL, NULL}; +static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; + +/* + * Create directory, also create parent directories if necessary. + */ +int +dir_create_dir(const char *dir, mode_t mode) +{ + char parent[MAXPGPATH]; + + strncpy(parent, dir, MAXPGPATH); + get_parent_directory(parent); + + /* Create parent first */ + if (access(parent, F_OK) == -1) + dir_create_dir(parent, mode); + + /* Create directory */ + if (mkdir(dir, mode) == -1) + { + if (errno == EEXIST) /* already exist */ + return 0; + elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); + } + + return 0; +} + +pgFile * +pgFileNew(const char *path, bool omit_symlink) +{ + struct stat st; + pgFile *file; + + /* stat the file */ + if ((omit_symlink ? stat(path, &st) : lstat(path, &st)) == -1) + { + /* file not found is not an error case */ + if (errno == ENOENT) + return NULL; + elog(ERROR, "cannot stat file \"%s\": %s", path, + strerror(errno)); + } + + file = pgFileInit(path); + file->size = st.st_size; + file->mode = st.st_mode; + + return file; +} + +pgFile * +pgFileInit(const char *path) +{ + pgFile *file; + char *file_name; + + file = (pgFile *) pgut_malloc(sizeof(pgFile)); + + file->name = NULL; + + file->size = 0; + file->mode = 0; + file->read_size = 0; + file->write_size = 0; + file->crc = 0; + file->is_datafile = false; + file->linked = NULL; + file->pagemap.bitmap = NULL; + file->pagemap.bitmapsize = PageBitmapIsEmpty; + file->pagemap_isabsent = false; + file->tblspcOid = 0; + file->dbOid = 0; + file->relOid = 0; + file->segno = 0; + file->is_database = false; + file->forkName = pgut_malloc(MAXPGPATH); + file->forkName[0] = '\0'; + + file->path = pgut_malloc(strlen(path) + 1); + strcpy(file->path, path); /* enough buffer size guaranteed */ + + /* Get file name from the path */ + file_name = strrchr(file->path, '/'); + if (file_name == NULL) + file->name = file->path; + else + { + file_name++; + file->name = file_name; + } + + file->is_cfs = false; + file->exists_in_prev = false; /* can change only in Incremental backup. */ + /* Number of blocks readed during backup */ + file->n_blocks = BLOCKNUM_INVALID; + file->compress_alg = NOT_DEFINED_COMPRESS; + return file; +} + +/* + * Delete file pointed by the pgFile. + * If the pgFile points directory, the directory must be empty. + */ +void +pgFileDelete(pgFile *file) +{ + if (S_ISDIR(file->mode)) + { + if (rmdir(file->path) == -1) + { + if (errno == ENOENT) + return; + else if (errno == ENOTDIR) /* could be symbolic link */ + goto delete_file; + + elog(ERROR, "cannot remove directory \"%s\": %s", + file->path, strerror(errno)); + } + return; + } + +delete_file: + if (remove(file->path) == -1) + { + if (errno == ENOENT) + return; + elog(ERROR, "cannot remove file \"%s\": %s", file->path, + strerror(errno)); + } +} + +pg_crc32 +pgFileGetCRC(const char *file_path) +{ + FILE *fp; + pg_crc32 crc = 0; + char buf[1024]; + size_t len; + int errno_tmp; + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", + file_path, strerror(errno)); + + /* calc CRC of backup file */ + INIT_CRC32C(crc); + while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf)) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + COMP_CRC32C(crc, buf, len); + } + errno_tmp = errno; + if (!feof(fp)) + elog(WARNING, "cannot read \"%s\": %s", file_path, + strerror(errno_tmp)); + if (len > 0) + COMP_CRC32C(crc, buf, len); + FIN_CRC32C(crc); + + fclose(fp); + + return crc; +} + +void +pgFileFree(void *file) +{ + pgFile *file_ptr; + + if (file == NULL) + return; + + file_ptr = (pgFile *) file; + + if (file_ptr->linked) + free(file_ptr->linked); + + if (file_ptr->forkName) + free(file_ptr->forkName); + + free(file_ptr->path); + free(file); +} + +/* Compare two pgFile with their path in ascending order of ASCII code. */ +int +pgFileComparePath(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->path, f2p->path); +} + +/* Compare two pgFile with their path in descending order of ASCII code. */ +int +pgFileComparePathDesc(const void *f1, const void *f2) +{ + return -pgFileComparePath(f1, f2); +} + +/* Compare two pgFile with their linked directory path. */ +int +pgFileCompareLinked(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->linked, f2p->linked); +} + +/* Compare two pgFile with their size */ +int +pgFileCompareSize(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + if (f1p->size > f2p->size) + return 1; + else if (f1p->size < f2p->size) + return -1; + else + return 0; +} + +static int +BlackListCompare(const void *str1, const void *str2) +{ + return strcmp(*(char **) str1, *(char **) str2); +} + +/* + * List files, symbolic links and directories in the directory "root" and add + * pgFile objects to "files". We add "root" to "files" if add_root is true. + * + * When omit_symlink is true, symbolic link is ignored and only file or + * directory llnked to will be listed. + */ +void +dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, + bool add_root) +{ + pgFile *file; + parray *black_list = NULL; + char path[MAXPGPATH]; + + join_path_components(path, backup_instance_path, PG_BLACK_LIST); + /* List files with black list */ + if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) + { + FILE *black_list_file = NULL; + char buf[MAXPGPATH * 2]; + char black_item[MAXPGPATH * 2]; + + black_list = parray_new(); + black_list_file = fopen(path, PG_BINARY_R); + + if (black_list_file == NULL) + elog(ERROR, "cannot open black_list: %s", strerror(errno)); + + while (fgets(buf, lengthof(buf), black_list_file) != NULL) + { + join_path_components(black_item, pgdata, buf); + + if (black_item[strlen(black_item) - 1] == '\n') + black_item[strlen(black_item) - 1] = '\0'; + + if (black_item[0] == '#' || black_item[0] == '\0') + continue; + + parray_append(black_list, black_item); + } + + fclose(black_list_file); + parray_qsort(black_list, BlackListCompare); + } + + file = pgFileNew(root, false); + if (file == NULL) + return; + + if (!S_ISDIR(file->mode)) + { + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + return; + } + if (add_root) + parray_append(files, file); + + dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); +} + +/* + * Check file or directory. + * + * Check for exclude. + * Extract information about the file parsing its name. + * Skip files: + * - skip temp tables files + * - skip unlogged tables files + * Set flags for: + * - database directories + * - datafiles + */ +static bool +dir_check_file(const char *root, pgFile *file) +{ + const char *rel_path; + int i; + int sscanf_res; + + /* Check if we need to exclude file by name */ + if (S_ISREG(file->mode)) + { + if (!exclusive_backup) + { + for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++) + if (strcmp(file->name, + pgdata_exclude_files_non_exclusive[i]) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding file: %s", file->name); + return false; + } + } + + for (i = 0; pgdata_exclude_files[i]; i++) + if (strcmp(file->name, pgdata_exclude_files[i]) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding file: %s", file->name); + return false; + } + } + /* + * If the directory name is in the exclude list, do not list the + * contents. + */ + else if (S_ISDIR(file->mode)) + { + /* + * If the item in the exclude list starts with '/', compare to + * the absolute path of the directory. Otherwise compare to the + * directory name portion. + */ + for (i = 0; pgdata_exclude_dir[i]; i++) + { + /* Full-path exclude*/ + if (pgdata_exclude_dir[i][0] == '/') + { + if (strcmp(file->path, pgdata_exclude_dir[i]) == 0) + { + elog(VERBOSE, "Excluding directory content: %s", + file->name); + return false; + } + } + else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) + { + elog(VERBOSE, "Excluding directory content: %s", + file->name); + return false; + } + } + } + + rel_path = GetRelativePath(file->path, root); + + /* + * Do not copy tablespaces twice. It may happen if the tablespace is located + * inside the PGDATA. + */ + if (S_ISDIR(file->mode) && + strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) + { + Oid tblspcOid; + char tmp_rel_path[MAXPGPATH]; + + /* + * Valid path for the tablespace is + * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + */ + if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + return false; + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s", + &tblspcOid, tmp_rel_path); + if (sscanf_res == 0) + return false; + } + + if (path_is_prefix_of_path("global", rel_path)) + { + file->tblspcOid = GLOBALTABLESPACE_OID; + + if (S_ISDIR(file->mode) && strcmp(file->name, "global") == 0) + file->is_database = true; + } + else if (path_is_prefix_of_path("base", rel_path)) + { + file->tblspcOid = DEFAULTTABLESPACE_OID; + + sscanf(rel_path, "base/%u/", &(file->dbOid)); + + if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) + file->is_database = true; + } + else if (path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + { + char tmp_rel_path[MAXPGPATH]; + + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", + &(file->tblspcOid), tmp_rel_path, + &(file->dbOid)); + + if (sscanf_res == 3 && S_ISDIR(file->mode) && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + file->is_database = true; + } + + /* Do not backup ptrack_init files */ + if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) + return false; + + /* + * Check files located inside database directories including directory + * 'global' + */ + if (S_ISREG(file->mode) && file->tblspcOid != 0 && + file->name && file->name[0]) + { + if (strcmp(file->name, "pg_internal.init") == 0) + return false; + /* Do not backup temp files */ + else if (file->name[0] == 't' && isdigit(file->name[1])) + return false; + else if (isdigit(file->name[0])) + { + char *fork_name; + int len; + char suffix[MAXPGPATH]; + + fork_name = strstr(file->name, "_"); + if (fork_name) + { + /* Auxiliary fork of the relfile */ + sscanf(file->name, "%u_%s", &(file->relOid), file->forkName); + + /* Do not backup ptrack files */ + if (strcmp(file->forkName, "ptrack") == 0) + return false; + } + else + { + len = strlen(file->name); + /* reloid.cfm */ + if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) + return true; + + sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), + &(file->segno), suffix); + if (sscanf_res == 0) + elog(ERROR, "Cannot parse file name \"%s\"", file->name); + else if (sscanf_res == 1 || sscanf_res == 2) + file->is_datafile = true; + } + } + } + + return true; +} + +/* + * List files in "root" directory. If "exclude" is true do not add into "files" + * files from pgdata_exclude_files and directories from pgdata_exclude_dir. + */ +static void +dir_list_file_internal(parray *files, const char *root, pgFile *parent, + bool exclude, bool omit_symlink, parray *black_list) +{ + DIR *dir; + struct dirent *dent; + + if (!S_ISDIR(parent->mode)) + elog(ERROR, "\"%s\" is not a directory", parent->path); + + /* Open directory and list contents */ + dir = opendir(parent->path); + if (dir == NULL) + { + if (errno == ENOENT) + { + /* Maybe the directory was removed */ + return; + } + elog(ERROR, "cannot open directory \"%s\": %s", + parent->path, strerror(errno)); + } + + errno = 0; + while ((dent = readdir(dir))) + { + pgFile *file; + char child[MAXPGPATH]; + + join_path_components(child, parent->path, dent->d_name); + + file = pgFileNew(child, omit_symlink); + if (file == NULL) + continue; + + /* Skip entries point current dir or parent dir */ + if (S_ISDIR(file->mode) && + (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) + { + pgFileFree(file); + continue; + } + + /* + * Add only files, directories and links. Skip sockets and other + * unexpected file formats. + */ + if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) + { + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + pgFileFree(file); + continue; + } + + /* Skip if the directory is in black_list defined by user */ + if (black_list && parray_bsearch(black_list, file->path, + BlackListCompare)) + { + elog(LOG, "Skip \"%s\": it is in the user's black list", file->path); + pgFileFree(file); + continue; + } + + /* We add the directory anyway */ + if (S_ISDIR(file->mode)) + parray_append(files, file); + + if (exclude && !dir_check_file(root, file)) + { + if (S_ISREG(file->mode)) + pgFileFree(file); + /* Skip */ + continue; + } + + /* At least add the file */ + if (S_ISREG(file->mode)) + parray_append(files, file); + + /* + * If the entry is a directory call dir_list_file_internal() + * recursively. + */ + if (S_ISDIR(file->mode)) + dir_list_file_internal(files, root, file, exclude, omit_symlink, + black_list); + } + + if (errno && errno != ENOENT) + { + int errno_tmp = errno; + closedir(dir); + elog(ERROR, "cannot read directory \"%s\": %s", + parent->path, strerror(errno_tmp)); + } + closedir(dir); +} + +/* + * List data directories excluding directories from + * pgdata_exclude_dir array. + * + * **is_root** is a little bit hack. We exclude only first level of directories + * and on the first level we check all files and directories. + */ +static void +list_data_directories(parray *files, const char *path, bool is_root, + bool exclude) +{ + DIR *dir; + struct dirent *dent; + int prev_errno; + bool has_child_dirs = false; + + /* open directory and list contents */ + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + bool skip = false; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + /* Check for exclude for the first level of listing */ + if (is_root && exclude) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++) + { + if (strcmp(dent->d_name, pgdata_exclude_dir[i]) == 0) + { + skip = true; + break; + } + } + } + if (skip) + continue; + + has_child_dirs = true; + list_data_directories(files, child, false, exclude); + } + + /* List only full and last directories */ + if (!is_root && !has_child_dirs) + { + pgFile *dir; + + dir = pgFileNew(path, false); + parray_append(files, dir); + } + + prev_errno = errno; + closedir(dir); + + if (prev_errno && prev_errno != ENOENT) + elog(ERROR, "cannot read directory \"%s\": %s", + path, strerror(prev_errno)); +} + +/* + * Save create directory path into memory. We can use it in next page restore to + * not raise the error "restore tablespace destination is not empty" in + * create_data_directories(). + */ +static void +set_tablespace_created(const char *link, const char *dir) +{ + TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); + + strcpy(cell->link_name, link); + strcpy(cell->linked_dir, dir); + cell->next = NULL; + + if (tablespace_created_dirs.tail) + tablespace_created_dirs.tail->next = cell; + else + tablespace_created_dirs.head = cell; + tablespace_created_dirs.tail = cell; +} + +/* + * Retrieve tablespace path, either relocated or original depending on whether + * -T was passed or not. + * + * Copy of function get_tablespace_mapping() from pg_basebackup.c. + */ +static const char * +get_tablespace_mapping(const char *dir) +{ + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(dir, cell->old_dir) == 0) + return cell->new_dir; + + return dir; +} + +/* + * Is directory was created when symlink was created in restore_directories(). + */ +static const char * +get_tablespace_created(const char *link) +{ + TablespaceCreatedListCell *cell; + + for (cell = tablespace_created_dirs.head; cell; cell = cell->next) + if (strcmp(link, cell->link_name) == 0) + return cell->linked_dir; + + return NULL; +} + +/* + * Split argument into old_dir and new_dir and append to tablespace mapping + * list. + * + * Copy of function tablespace_list_append() from pg_basebackup.c. + */ +void +opt_tablespace_map(pgut_option *opt, const char *arg) +{ + TablespaceListCell *cell = pgut_new(TablespaceListCell); + char *dst; + char *dst_ptr; + const char *arg_ptr; + + dst_ptr = dst = cell->old_dir; + for (arg_ptr = arg; *arg_ptr; arg_ptr++) + { + if (dst_ptr - dst >= MAXPGPATH) + elog(ERROR, "directory name too long"); + + if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') + ; /* skip backslash escaping = */ + else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) + { + if (*cell->new_dir) + elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); + else + dst = dst_ptr = cell->new_dir; + } + else + *dst_ptr++ = *arg_ptr; + } + + if (!*cell->old_dir || !*cell->new_dir) + elog(ERROR, "invalid tablespace mapping format \"%s\", " + "must be \"OLDDIR=NEWDIR\"", arg); + + /* + * This check isn't absolutely necessary. But all tablespaces are created + * with absolute directories, so specifying a non-absolute path here would + * just never match, possibly confusing users. It's also good to be + * consistent with the new_dir check. + */ + if (!is_absolute_path(cell->old_dir)) + elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", + cell->old_dir); + + if (!is_absolute_path(cell->new_dir)) + elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", + cell->new_dir); + + if (tablespace_dirs.tail) + tablespace_dirs.tail->next = cell; + else + tablespace_dirs.head = cell; + tablespace_dirs.tail = cell; +} + +/* + * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise + * an error if target directories exist. + * + * If **extract_tablespaces** is true then try to extract tablespace data + * directories into their initial path using tablespace_map file. + */ +void +create_data_directories(const char *data_dir, const char *backup_dir, + bool extract_tablespaces) +{ + parray *dirs, + *links = NULL; + size_t i; + char backup_database_dir[MAXPGPATH], + to_path[MAXPGPATH]; + + dirs = parray_new(); + if (extract_tablespaces) + { + links = parray_new(); + read_tablespace_map(links, backup_dir); + } + + join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); + list_data_directories(dirs, backup_database_dir, true, false); + + elog(LOG, "restore directories and symlinks..."); + + for (i = 0; i < parray_num(dirs); i++) + { + pgFile *dir = (pgFile *) parray_get(dirs, i); + char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); + + Assert(S_ISDIR(dir->mode)); + + /* Try to create symlink and linked directory if necessary */ + if (extract_tablespaces && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) + { + char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), + *link_sep, + *tmp_ptr; + char link_name[MAXPGPATH]; + pgFile **link; + + /* Extract link name from relative path */ + link_sep = first_dir_separator(link_ptr); + if (link_sep != NULL) + { + int len = link_sep - link_ptr; + strncpy(link_name, link_ptr, len); + link_name[len] = '\0'; + } + else + goto create_directory; + + tmp_ptr = dir->path; + dir->path = link_name; + /* Search only by symlink name without path */ + link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); + dir->path = tmp_ptr; + + if (link) + { + const char *linked_path = get_tablespace_mapping((*link)->linked); + const char *dir_created; + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + /* Check if linked directory was created earlier */ + dir_created = get_tablespace_created(link_name); + if (dir_created) + { + /* + * If symlink and linked directory were created do not + * create it second time. + */ + if (strcmp(dir_created, linked_path) == 0) + { + /* + * Create rest of directories. + * First check is there any directory name after + * separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + else + continue; + } + else + elog(ERROR, "tablespace directory \"%s\" of page backup does not " + "match with previous created tablespace directory \"%s\" of symlink \"%s\"", + linked_path, dir_created, link_name); + } + + /* + * This check was done in check_tablespace_mapping(). But do + * it again. + */ + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + + if (link_sep) + elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", + linked_path, + (int) (link_sep - relative_ptr), relative_ptr); + else + elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", + linked_path, relative_ptr); + + /* Firstly, create linked directory */ + dir_create_dir(linked_path, DIR_PERMISSION); + + join_path_components(to_path, data_dir, PG_TBLSPC_DIR); + /* Create pg_tblspc directory just in case */ + dir_create_dir(to_path, DIR_PERMISSION); + + /* Secondly, create link */ + join_path_components(to_path, to_path, link_name); + if (symlink(linked_path, to_path) < 0) + elog(ERROR, "could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + /* Save linked directory */ + set_tablespace_created(link_name, linked_path); + + /* + * Create rest of directories. + * First check is there any directory name after separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + + continue; + } + } + +create_directory: + elog(LOG, "create directory \"%s\"", relative_ptr); + + /* This is not symlink, create directory */ + join_path_components(to_path, data_dir, relative_ptr); + dir_create_dir(to_path, DIR_PERMISSION); + } + + if (extract_tablespaces) + { + parray_walk(links, pgFileFree); + parray_free(links); + } + + parray_walk(dirs, pgFileFree); + parray_free(dirs); +} + +/* + * Read names of symbolik names of tablespaces with links to directories from + * tablespace_map or tablespace_map.txt. + */ +void +read_tablespace_map(parray *files, const char *backup_dir) +{ + FILE *fp; + char db_path[MAXPGPATH], + map_path[MAXPGPATH]; + char buf[MAXPGPATH * 2]; + + join_path_components(db_path, backup_dir, DATABASE_DIR); + join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); + + /* Exit if database/tablespace_map doesn't exist */ + if (!fileExists(map_path)) + { + elog(LOG, "there is no file tablespace_map"); + return; + } + + fp = fopen(map_path, "rt"); + if (fp == NULL) + elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno)); + + while (fgets(buf, lengthof(buf), fp)) + { + char link_name[MAXPGPATH], + path[MAXPGPATH]; + pgFile *file; + + if (sscanf(buf, "%1023s %1023s", link_name, path) != 2) + elog(ERROR, "invalid format found in \"%s\"", map_path); + + file = pgut_new(pgFile); + memset(file, 0, sizeof(pgFile)); + + file->path = pgut_malloc(strlen(link_name) + 1); + strcpy(file->path, link_name); + + file->linked = pgut_malloc(strlen(path) + 1); + strcpy(file->linked, path); + + parray_append(files, file); + } + + parray_qsort(files, pgFileCompareLinked); + fclose(fp); +} + +/* + * Check that all tablespace mapping entries have correct linked directory + * paths. Linked directories must be empty or do not exist. + * + * If tablespace-mapping option is supplied, all OLDDIR entries must have + * entries in tablespace_map file. + */ +void +check_tablespace_mapping(pgBackup *backup) +{ + char this_backup_path[MAXPGPATH]; + parray *links; + size_t i; + TablespaceListCell *cell; + pgFile *tmp_file = pgut_new(pgFile); + + links = parray_new(); + + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, this_backup_path); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "check tablespace directories of backup %s", + base36enc(backup->start_time)); + + /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ + for (cell = tablespace_dirs.head; cell; cell = cell->next) + { + tmp_file->linked = cell->old_dir; + + if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) + elog(ERROR, "--tablespace-mapping option's old directory " + "doesn't have an entry in tablespace_map file: \"%s\"", + cell->old_dir); + } + + /* 2 - all linked directories must be empty */ + for (i = 0; i < parray_num(links); i++) + { + pgFile *link = (pgFile *) parray_get(links, i); + const char *linked_path = link->linked; + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(link->linked, cell->old_dir) == 0) + { + linked_path = cell->new_dir; + break; + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + } + + free(tmp_file); + parray_walk(links, pgFileFree); + parray_free(links); +} + +/* + * Print backup content list. + */ +void +print_file_list(FILE *out, const parray *files, const char *root) +{ + size_t i; + + /* print each file in the list */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *path = file->path; + + /* omit root directory portion */ + if (root && strstr(path, root) == path) + path = GetRelativePath(path, root); + + fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"is_cfs\":\"%u\", \"crc\":\"%u\", " + "\"compress_alg\":\"%s\"", + path, file->write_size, file->mode, + file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, + deparse_compress_alg(file->compress_alg)); + + if (file->is_datafile) + fprintf(out, ",\"segno\":\"%d\"", file->segno); + +#ifndef WIN32 + if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif + fprintf(out, ",\"linked\":\"%s\"", file->linked); + + if (file->n_blocks != BLOCKNUM_INVALID) + fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); + + fprintf(out, "}\n"); + } +} + +/* Parsing states for get_control_value() */ +#define CONTROL_WAIT_NAME 1 +#define CONTROL_INNAME 2 +#define CONTROL_WAIT_COLON 3 +#define CONTROL_WAIT_VALUE 4 +#define CONTROL_INVALUE 5 +#define CONTROL_WAIT_NEXT_NAME 6 + +/* + * Get value from json-like line "str" of backup_content.control file. + * + * The line has the following format: + * {"name1":"value1", "name2":"value2"} + * + * The value will be returned to "value_str" as string if it is not NULL. If it + * is NULL the value will be returned to "value_int64" as int64. + * + * Returns true if the value was found in the line. + */ +static bool +get_control_value(const char *str, const char *name, + char *value_str, int64 *value_int64, bool is_mandatory) +{ + int state = CONTROL_WAIT_NAME; + char *name_ptr = (char *) name; + char *buf = (char *) str; + char buf_int64[32], /* Buffer for "value_int64" */ + *buf_int64_ptr = buf_int64; + + /* Set default values */ + if (value_str) + *value_str = '\0'; + else if (value_int64) + *value_int64 = 0; + + while (*buf) + { + switch (state) + { + case CONTROL_WAIT_NAME: + if (*buf == '"') + state = CONTROL_INNAME; + else if (IsAlpha(*buf)) + goto bad_format; + break; + case CONTROL_INNAME: + /* Found target field. Parse value. */ + if (*buf == '"') + state = CONTROL_WAIT_COLON; + /* Check next field */ + else if (*buf != *name_ptr) + { + name_ptr = (char *) name; + state = CONTROL_WAIT_NEXT_NAME; + } + else + name_ptr++; + break; + case CONTROL_WAIT_COLON: + if (*buf == ':') + state = CONTROL_WAIT_VALUE; + else if (!IsSpace(*buf)) + goto bad_format; + break; + case CONTROL_WAIT_VALUE: + if (*buf == '"') + { + state = CONTROL_INVALUE; + buf_int64_ptr = buf_int64; + } + else if (IsAlpha(*buf)) + goto bad_format; + break; + case CONTROL_INVALUE: + /* Value was parsed, exit */ + if (*buf == '"') + { + if (value_str) + { + *value_str = '\0'; + } + else if (value_int64) + { + /* Length of buf_uint64 should not be greater than 31 */ + if (buf_int64_ptr - buf_int64 >= 32) + elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); + + *buf_int64_ptr = '\0'; + if (!parse_int64(buf_int64, value_int64, 0)) + goto bad_format; + } + + return true; + } + else + { + if (value_str) + { + *value_str = *buf; + value_str++; + } + else + { + *buf_int64_ptr = *buf; + buf_int64_ptr++; + } + } + break; + case CONTROL_WAIT_NEXT_NAME: + if (*buf == ',') + state = CONTROL_WAIT_NAME; + break; + default: + /* Should not happen */ + break; + } + + buf++; + } + + /* There is no close quotes */ + if (state == CONTROL_INNAME || state == CONTROL_INVALUE) + goto bad_format; + + /* Did not find target field */ + if (is_mandatory) + elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); + return false; + +bad_format: + elog(ERROR, "%s file has invalid format in line %s", + DATABASE_FILE_LIST, str); + return false; /* Make compiler happy */ +} + +/* + * Construct parray of pgFile from the backup content list. + * If root is not NULL, path will be absolute path. + */ +parray * +dir_read_file_list(const char *root, const char *file_txt) +{ + FILE *fp; + parray *files; + char buf[MAXPGPATH * 2]; + + fp = fopen(file_txt, "rt"); + if (fp == NULL) + elog(errno == ENOENT ? ERROR : ERROR, + "cannot open \"%s\": %s", file_txt, strerror(errno)); + + files = parray_new(); + + while (fgets(buf, lengthof(buf), fp)) + { + char path[MAXPGPATH]; + char filepath[MAXPGPATH]; + char linked[MAXPGPATH]; + char compress_alg_string[MAXPGPATH]; + int64 write_size, + mode, /* bit length of mode_t depends on platforms */ + is_datafile, + is_cfs, + crc, + segno, + n_blocks; + pgFile *file; + + get_control_value(buf, "path", path, NULL, true); + get_control_value(buf, "size", NULL, &write_size, true); + get_control_value(buf, "mode", NULL, &mode, true); + get_control_value(buf, "is_datafile", NULL, &is_datafile, true); + get_control_value(buf, "is_cfs", NULL, &is_cfs, false); + get_control_value(buf, "crc", NULL, &crc, true); + get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); + + if (root) + join_path_components(filepath, root, path); + else + strcpy(filepath, path); + + file = pgFileInit(filepath); + + file->write_size = (int64) write_size; + file->mode = (mode_t) mode; + file->is_datafile = is_datafile ? true : false; + file->is_cfs = is_cfs ? true : false; + file->crc = (pg_crc32) crc; + file->compress_alg = parse_compress_alg(compress_alg_string); + + /* + * Optional fields + */ + + if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) + file->linked = pgut_strdup(linked); + + if (get_control_value(buf, "segno", NULL, &segno, false)) + file->segno = (int) segno; + + if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) + file->n_blocks = (int) n_blocks; + + parray_append(files, file); + } + + fclose(fp); + return files; +} + +/* + * Check if directory empty. + */ +bool +dir_is_empty(const char *path) +{ + DIR *dir; + struct dirent *dir_ent; + + dir = opendir(path); + if (dir == NULL) + { + /* Directory in path doesn't exist */ + if (errno == ENOENT) + return true; + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + } + + errno = 0; + while ((dir_ent = readdir(dir))) + { + /* Skip entries point current dir or parent dir */ + if (strcmp(dir_ent->d_name, ".") == 0 || + strcmp(dir_ent->d_name, "..") == 0) + continue; + + /* Directory is not empty */ + closedir(dir); + return false; + } + if (errno) + elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno)); + + closedir(dir); + + return true; +} + +/* + * Return true if the path is a existing regular file. + */ +bool +fileExists(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1 && errno == ENOENT) + return false; + else if (!S_ISREG(buf.st_mode)) + return false; + else + return true; +} + +size_t +pgFileSize(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + + return buf.st_size; +} diff --git a/src/fetch.c b/src/fetch.c new file mode 100644 index 00000000..0d4dbdaa --- /dev/null +++ b/src/fetch.c @@ -0,0 +1,116 @@ +/*------------------------------------------------------------------------- + * + * fetch.c + * Functions for fetching files from PostgreSQL data directory + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "catalog/catalog.h" + +#include +#include +#include +#include +#include +#include + +#include "pg_probackup.h" + +/* + * Read a file into memory. The file to be read is /. + * The file contents are returned in a malloc'd buffer, and *filesize + * is set to the length of the file. + * + * The returned buffer is always zero-terminated; the size of the returned + * buffer is actually *filesize + 1. That's handy when reading a text file. + * This function can be used to read binary files as well, you can just + * ignore the zero-terminator in that case. + * + */ +char * +slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) +{ + int fd; + char *buffer; + struct stat statbuf; + char fullpath[MAXPGPATH]; + int len; + snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); + + if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + + if (fstat(fd, &statbuf) < 0) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + + len = statbuf.st_size; + + buffer = pg_malloc(len + 1); + + if (read(fd, buffer, len) != len) + { + if (safe) + return NULL; + else + elog(ERROR, "could not read file \"%s\": %s\n", + fullpath, strerror(errno)); + } + + close(fd); + + /* Zero-terminate the buffer. */ + buffer[len] = '\0'; + + if (filesize) + *filesize = len; + return buffer; +} + +/* + * Receive a single file as a malloc'd buffer. + */ +char * +fetchFile(PGconn *conn, const char *filename, size_t *filesize) +{ + PGresult *res; + char *result; + const char *params[1]; + int len; + + params[0] = filename; + res = pgut_execute_extended(conn, "SELECT pg_catalog.pg_read_binary_file($1)", + 1, params, false, false); + + /* sanity check the result set */ + if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) + elog(ERROR, "unexpected result set while fetching remote file \"%s\"", + filename); + + /* Read result to local variables */ + len = PQgetlength(res, 0, 0); + result = pg_malloc(len + 1); + memcpy(result, PQgetvalue(res, 0, 0), len); + result[len] = '\0'; + + PQclear(res); + *filesize = len; + + return result; +} diff --git a/src/help.c b/src/help.c new file mode 100644 index 00000000..dc9cc3d8 --- /dev/null +++ b/src/help.c @@ -0,0 +1,605 @@ +/*------------------------------------------------------------------------- + * + * help.c + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +static void help_init(void); +static void help_backup(void); +static void help_restore(void); +static void help_validate(void); +static void help_show(void); +static void help_delete(void); +static void help_merge(void); +static void help_set_config(void); +static void help_show_config(void); +static void help_add_instance(void); +static void help_del_instance(void); +static void help_archive_push(void); +static void help_archive_get(void); + +void +help_command(char *command) +{ + if (strcmp(command, "init") == 0) + help_init(); + else if (strcmp(command, "backup") == 0) + help_backup(); + else if (strcmp(command, "restore") == 0) + help_restore(); + else if (strcmp(command, "validate") == 0) + help_validate(); + else if (strcmp(command, "show") == 0) + help_show(); + else if (strcmp(command, "delete") == 0) + help_delete(); + else if (strcmp(command, "merge") == 0) + help_merge(); + else if (strcmp(command, "set-config") == 0) + help_set_config(); + else if (strcmp(command, "show-config") == 0) + help_show_config(); + else if (strcmp(command, "add-instance") == 0) + help_add_instance(); + else if (strcmp(command, "del-instance") == 0) + help_del_instance(); + else if (strcmp(command, "archive-push") == 0) + help_archive_push(); + else if (strcmp(command, "archive-get") == 0) + help_archive_get(); + else if (strcmp(command, "--help") == 0 + || strcmp(command, "help") == 0 + || strcmp(command, "-?") == 0 + || strcmp(command, "--version") == 0 + || strcmp(command, "version") == 0 + || strcmp(command, "-V") == 0) + printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); + else + printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); + exit(0); +} + +void +help_pg_probackup(void) +{ + printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + + printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); + + printf(_("\n %s version\n"), PROGRAM_NAME); + + printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); + + printf(_("\n %s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--archive-timeout=timeout]\n")); + + printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n")); + + printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); + printf(_(" [--restore-as-replica]\n")); + printf(_(" [--no-validate]\n")); + + printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); + printf(_(" [--timeline=timeline]\n")); + + printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n")); + + printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--wal] [-i backup-id | --expired]\n")); + printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id\n")); + + printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--overwrite]\n")); + + printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + + if ((PROGRAM_URL || PROGRAM_EMAIL)) + { + printf("\n"); + if (PROGRAM_URL) + printf("Read the website for details. <%s>\n", PROGRAM_URL); + if (PROGRAM_EMAIL) + printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); + } + exit(0); +} + +static void +help_init(void) +{ + printf(_("%s init -B backup-path\n\n"), PROGRAM_NAME); + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); +} + +static void +help_backup(void) +{ + printf(_("%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); + printf(_(" --stream stream the transaction log and include it in the backup\n")); + printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --backup-pg-log backup of pg_log directory\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --delete-expired delete backups expired according to current\n")); + printf(_(" retention policy after successful backup completion\n")); + printf(_(" --delete-wal remove redundant archived wal files\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress compress data files\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib', 'pglz', 'none' (default: zlib)\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); +} + +static void +help_restore(void) +{ + printf(_("%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); + printf(_(" [--restore-as-replica] [--no-validate]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" -i, --backup-id=backup-id backup to restore\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + + printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); + printf(_(" --recovery-target-action=pause|promote|shutdown\n")); + printf(_(" action the server should take once the recovery target is reached\n")); + printf(_(" (default: pause)\n")); + + printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); + printf(_(" to ease setting up a standby server\n")); + printf(_(" --no-validate disable backup validation during restore\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_validate(void) +{ + printf(_("%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to validate\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_show(void) +{ + printf(_("%s show -B backup-path\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name show info about specific intstance\n")); + printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); +} + +static void +help_delete(void) +{ + printf(_("%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-i backup-id | --expired] [--wal]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to delete\n")); + printf(_(" --expired delete backups expired according to current\n")); + printf(_(" retention policy\n")); + printf(_(" --wal remove unnecessary wal files in WAL ARCHIVE\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_merge(void) +{ + printf(_("%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id [-j num-threads] [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to merge\n")); + + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_set_config(void) +{ + printf(_("%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--archive-timeout=timeout]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); +} + +static void +help_show_config(void) +{ + printf(_("%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); +} + +static void +help_add_instance(void) +{ + printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" --instance=instance_name name of the new instance\n")); +} + +static void +help_del_instance(void) +{ + printf(_("%s del-instance -B backup-path --instance=instance_name\n\n"), PROGRAM_NAME); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); +} + +static void +help_archive_push(void) +{ + printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--overwrite]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative path name of the WAL file on the server\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" name of the WAL file to retrieve from the server\n")); + printf(_(" --compress compress WAL file during archiving\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + printf(_(" --overwrite overwrite archived WAL file\n")); +} + +static void +help_archive_get(void) +{ + printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative destination path name of the WAL file on the server\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" name of the WAL file to retrieve from the archive\n")); +} diff --git a/src/init.c b/src/init.c new file mode 100644 index 00000000..cd559cb4 --- /dev/null +++ b/src/init.c @@ -0,0 +1,108 @@ +/*------------------------------------------------------------------------- + * + * init.c: - initialize backup catalog. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +/* + * Initialize backup catalog. + */ +int +do_init(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + int results; + + results = pg_check_dir(backup_path); + if (results == 4) /* exists and not empty*/ + elog(ERROR, "backup catalog already exist and it's not empty"); + else if (results == -1) /*trouble accessing directory*/ + { + int errno_tmp = errno; + elog(ERROR, "cannot open backup catalog directory \"%s\": %s", + backup_path, strerror(errno_tmp)); + } + + /* create backup catalog root directory */ + dir_create_dir(backup_path, DIR_PERMISSION); + + /* create backup catalog data directory */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir_create_dir(path, DIR_PERMISSION); + + /* create backup catalog wal directory */ + join_path_components(arclog_path_dir, backup_path, "wal"); + dir_create_dir(arclog_path_dir, DIR_PERMISSION); + + elog(INFO, "Backup catalog '%s' successfully inited", backup_path); + return 0; +} + +int +do_add_instance(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + struct stat st; + pgBackupConfig *config = pgut_new(pgBackupConfig); + + /* PGDATA is always required */ + if (pgdata == NULL) + elog(ERROR, "Required parameter not specified: PGDATA " + "(-D, --pgdata)"); + + /* Read system_identifier from PGDATA */ + system_identifier = get_system_identifier(pgdata); + /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ + xlog_seg_size = get_xlog_seg_size(pgdata); + + /* Ensure that all root directories already exist */ + if (access(backup_path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", backup_path); + + join_path_components(path, backup_path, BACKUPS_DIR); + if (access(path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", path); + + join_path_components(arclog_path_dir, backup_path, "wal"); + if (access(arclog_path_dir, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", arclog_path_dir); + + /* Create directory for data files of this specific instance */ + if (stat(backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "instance '%s' already exists", backup_instance_path); + dir_create_dir(backup_instance_path, DIR_PERMISSION); + + /* + * Create directory for wal files of this specific instance. + * Existence check is extra paranoid because if we don't have such a + * directory in data dir, we shouldn't have it in wal as well. + */ + if (stat(arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "arclog_path '%s' already exists", arclog_path); + dir_create_dir(arclog_path, DIR_PERMISSION); + + /* + * Wite initial config. system-identifier and pgdata are set in + * init subcommand and will never be updated. + */ + pgBackupConfigInit(config); + config->system_identifier = system_identifier; + config->xlog_seg_size = xlog_seg_size; + config->pgdata = pgdata; + writeBackupCatalogConfigFile(config); + + elog(INFO, "Instance '%s' successfully inited", instance_name); + return 0; +} diff --git a/src/merge.c b/src/merge.c new file mode 100644 index 00000000..979a1729 --- /dev/null +++ b/src/merge.c @@ -0,0 +1,526 @@ +/*------------------------------------------------------------------------- + * + * merge.c: merge FULL and incremental backups + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +typedef struct +{ + parray *to_files; + parray *files; + + pgBackup *to_backup; + pgBackup *from_backup; + + const char *to_root; + const char *from_root; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} merge_files_arg; + +static void merge_backups(pgBackup *backup, pgBackup *next_backup); +static void *merge_files(void *arg); + +/* + * Implementation of MERGE command. + * + * - Find target and its parent full backup + * - Merge data files of target, parent and and intermediate backups + * - Remove unnecessary files, which doesn't exist in the target backup anymore + */ +void +do_merge(time_t backup_id) +{ + parray *backups; + pgBackup *dest_backup = NULL; + pgBackup *full_backup = NULL; + time_t prev_parent = INVALID_BACKUP_ID; + int i; + int dest_backup_idx = 0; + int full_backup_idx = 0; + + if (backup_id == INVALID_BACKUP_ID) + elog(ERROR, "required parameter is not specified: --backup-id"); + + if (instance_name == NULL) + elog(ERROR, "required parameter is not specified: --instance"); + + elog(LOG, "Merge started"); + + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find destination and parent backups */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->start_time > backup_id) + continue; + else if (backup->start_time == backup_id && !dest_backup) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + if (backup->backup_mode == BACKUP_MODE_FULL) + elog(ERROR, "Backup %s if full backup", + base36enc(backup->start_time)); + + dest_backup = backup; + dest_backup_idx = i; + } + else + { + Assert(dest_backup); + + if (backup->start_time != prev_parent) + continue; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Skipping backup %s, because it has non-valid status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + /* If we already found dest_backup, look for full backup */ + if (dest_backup && backup->backup_mode == BACKUP_MODE_FULL) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Parent full backup %s for the given backup %s has status: %s", + base36enc_dup(backup->start_time), + base36enc_dup(dest_backup->start_time), + status2str(backup->status)); + + full_backup = backup; + full_backup_idx = i; + + /* Found target and full backups, so break the loop */ + break; + } + } + + prev_parent = backup->parent_backup; + } + + if (dest_backup == NULL) + elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); + if (full_backup == NULL) + elog(ERROR, "Parent full backup for the given backup %s was not found", + base36enc(backup_id)); + + Assert(full_backup_idx != dest_backup_idx); + + /* + * Found target and full backups, merge them and intermediate backups + */ + for (i = full_backup_idx; i > dest_backup_idx; i--) + { + pgBackup *to_backup = (pgBackup *) parray_get(backups, i); + pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); + + merge_backups(to_backup, from_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(LOG, "Merge completed"); +} + +/* + * Merge two backups data files using threads. + * - move instance files from from_backup to to_backup + * - remove unnecessary directories and files from to_backup + * - update metadata of from_backup, it becames FULL backup + */ +static void +merge_backups(pgBackup *to_backup, pgBackup *from_backup) +{ + char *to_backup_id = base36enc_dup(to_backup->start_time), + *from_backup_id = base36enc_dup(from_backup->start_time); + char to_backup_path[MAXPGPATH], + to_database_path[MAXPGPATH], + from_backup_path[MAXPGPATH], + from_database_path[MAXPGPATH], + control_file[MAXPGPATH]; + parray *files, + *to_files; + pthread_t *threads; + merge_files_arg *threads_args; + int i; + bool merge_isok = true; + + elog(LOG, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + + to_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(to_backup); + + from_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(from_backup); + + /* + * Make backup paths. + */ + pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); + pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), + DATABASE_DIR); + pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); + pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), + DATABASE_DIR); + + create_data_directories(to_database_path, from_backup_path, false); + + /* + * Get list of files which will be modified or removed. + */ + pgBackupGetPath(to_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + to_files = dir_read_file_list(from_database_path, /* Use from_database_path + * so root path will be + * equal with 'files' */ + control_file); + /* To delete from leaf, sort in reversed order */ + parray_qsort(to_files, pgFileComparePathDesc); + /* + * Get list of files which need to be moved. + */ + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + files = dir_read_file_list(from_database_path, control_file); + /* sort by size for load balancing */ + parray_qsort(files, pgFileCompareSize); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + + /* Setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_init_flag(&file->lock); + } + + for (i = 0; i < num_threads; i++) + { + merge_files_arg *arg = &(threads_args[i]); + + arg->to_files = to_files; + arg->files = files; + arg->to_backup = to_backup; + arg->from_backup = from_backup; + arg->to_root = to_database_path; + arg->from_root = from_database_path; + /* By default there are some error */ + arg->ret = 1; + + elog(VERBOSE, "Start thread: %d", i); + + pthread_create(&threads[i], NULL, merge_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + merge_isok = false; + } + if (!merge_isok) + elog(ERROR, "Data files merging failed"); + + /* + * Files were copied into to_backup and deleted from from_backup. Remove + * remaining directories from from_backup. + */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (!S_ISDIR(file->mode)) + continue; + + if (rmdir(file->path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + file->path, strerror(errno)); + } + if (rmdir(from_database_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_database_path, strerror(errno)); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + BACKUP_CONTROL_FILE); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + if (rmdir(from_backup_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_backup_path, strerror(errno)); + + /* + * Delete files which are not in from_backup file list. + */ + for (i = 0; i < parray_num(to_files); i++) + { + pgFile *file = (pgFile *) parray_get(to_files, i); + + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + elog(LOG, "Deleted \"%s\"", file->path); + } + } + + /* + * Rename FULL backup directory. + */ + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Update to_backup metadata. + */ + pgBackupCopy(to_backup, from_backup); + /* Correct metadata */ + to_backup->backup_mode = BACKUP_MODE_FULL; + to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + /* Compute summary of size of regular files in the backup */ + to_backup->data_bytes = 0; + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (S_ISDIR(file->mode)) + to_backup->data_bytes += 4096; + /* Count the amount of the data actually copied */ + else if (S_ISREG(file->mode)) + to_backup->data_bytes += file->write_size; + } + /* compute size of wal files of this backup stored in the archive */ + if (!to_backup->stream) + to_backup->wal_bytes = xlog_seg_size * + (to_backup->stop_lsn / xlog_seg_size - + to_backup->start_lsn / xlog_seg_size + 1); + else + to_backup->wal_bytes = BYTES_INVALID; + + pgBackupWriteFileList(to_backup, files, from_database_path); + pgBackupWriteBackupControlFile(to_backup); + + /* Cleanup */ + pfree(threads_args); + pfree(threads); + + parray_walk(to_files, pgFileFree); + parray_free(to_files); + + parray_walk(files, pgFileFree); + parray_free(files); + + pfree(to_backup_id); + pfree(from_backup_id); +} + +/* + * Thread worker of merge_backups(). + */ +static void * +merge_files(void *arg) +{ + merge_files_arg *argument = (merge_files_arg *) arg; + pgBackup *to_backup = argument->to_backup; + pgBackup *from_backup = argument->from_backup; + char tmp_file_path[MAXPGPATH]; + int i, + num_files = parray_num(argument->files); + int to_root_len = strlen(argument->to_root); + + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + join_path_components(tmp_file_path, argument->to_root, "tmp"); + + for (i = 0; i < num_files; i++) + { + pgFile *file = (pgFile *) parray_get(argument->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during merging backups"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); + + /* + * Skip files which haven't changed since previous backup. But in case + * of DELTA backup we should consider n_blocks to truncate the target + * backup. + */ + if (file->write_size == BYTES_INVALID && + file->n_blocks == -1) + { + elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", + file->path); + + /* + * If the file wasn't changed in PAGE backup, retreive its + * write_size from previous FULL backup. + */ + if (S_ISREG(file->mode)) + { + pgFile **res_file; + + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + if (res_file && *res_file) + { + file->compress_alg = (*res_file)->compress_alg; + file->write_size = (*res_file)->write_size; + file->crc = (*res_file)->crc; + } + } + + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + + /* + * Move the file. We need to decompress it and compress again if + * necessary. + */ + elog(VERBOSE, "Moving file \"%s\", is_datafile %d, is_cfs %d", + file->path, file->is_database, file->is_cfs); + + if (file->is_datafile && !file->is_cfs) + { + char to_path_tmp[MAXPGPATH]; /* Path of target file */ + + join_path_components(to_path_tmp, argument->to_root, + file->path + to_root_len + 1); + + /* + * We need more complicate algorithm if target file exists and it is + * compressed. + */ + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + { + char *prev_path; + + /* Start the magic */ + + /* + * Merge files: + * - decompress first file + * - decompress second file and merge with first decompressed file + * - compress result file + */ + + elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", + tmp_file_path); + + prev_path = file->path; + /* + * We need to decompress target file only if it exists. + */ + if (fileExists(to_path_tmp)) + { + /* + * file->path points to the file in from_root directory. But we + * need the file in directory to_root. + */ + file->path = to_path_tmp; + + /* Decompress first/target file */ + restore_data_file(tmp_file_path, file, false, false); + + file->path = prev_path; + } + /* Merge second/source file with first/target file */ + restore_data_file(tmp_file_path, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + + elog(VERBOSE, "Compress file and save it to the directory \"%s\"", + argument->to_root); + + /* Again we need change path */ + file->path = tmp_file_path; + /* backup_data_file() requires file size to calculate nblocks */ + file->size = pgFileSize(file->path); + /* Now we can compress the file */ + backup_data_file(NULL, /* We shouldn't need 'arguments' here */ + to_path_tmp, file, + to_backup->start_lsn, + to_backup->backup_mode, + to_backup->compress_alg, + to_backup->compress_level); + + file->path = prev_path; + + /* We can remove temporary file now */ + if (unlink(tmp_file_path)) + elog(ERROR, "Could not remove temporary file \"%s\": %s", + tmp_file_path, strerror(errno)); + } + /* + * Otherwise merging algorithm is simpler. + */ + else + { + /* We can merge in-place here */ + restore_data_file(to_path_tmp, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + true); + + /* + * We need to calculate write_size, restore_data_file() doesn't + * do that. + */ + file->write_size = pgFileSize(to_path_tmp); + file->crc = pgFileGetCRC(to_path_tmp); + } + pgFileDelete(file); + } + else + move_file(argument->from_root, argument->to_root, file); + + if (file->write_size != BYTES_INVALID) + elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files merging is successful */ + argument->ret = 0; + + return NULL; +} diff --git a/src/parsexlog.c b/src/parsexlog.c new file mode 100644 index 00000000..297269b6 --- /dev/null +++ b/src/parsexlog.c @@ -0,0 +1,1039 @@ +/*------------------------------------------------------------------------- + * + * parsexlog.c + * Functions for reading Write-Ahead-Log + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#ifdef HAVE_LIBZ +#include +#endif + +#include "commands/dbcommands_xlog.h" +#include "catalog/storage_xlog.h" +#include "access/transam.h" +#include "utils/thread.h" + +/* + * RmgrNames is an array of resource manager names, to make error messages + * a bit nicer. + */ +#if PG_VERSION_NUM >= 100000 +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ + name, +#else +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \ + name, +#endif + +static const char *RmgrNames[RM_MAX_ID + 1] = { +#include "access/rmgrlist.h" +}; + +/* some from access/xact.h */ +/* + * XLOG allows to store some information in high 4 bits of log record xl_info + * field. We use 3 for the opcode, and one about an optional flag variable. + */ +#define XLOG_XACT_COMMIT 0x00 +#define XLOG_XACT_PREPARE 0x10 +#define XLOG_XACT_ABORT 0x20 +#define XLOG_XACT_COMMIT_PREPARED 0x30 +#define XLOG_XACT_ABORT_PREPARED 0x40 +#define XLOG_XACT_ASSIGNMENT 0x50 +/* free opcode 0x60 */ +/* free opcode 0x70 */ + +/* mask for filtering opcodes out of xl_info */ +#define XLOG_XACT_OPMASK 0x70 + +typedef struct xl_xact_commit +{ + TimestampTz xact_time; /* time of commit */ + + /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ + /* xl_xact_dbinfo follows if XINFO_HAS_DBINFO */ + /* xl_xact_subxacts follows if XINFO_HAS_SUBXACT */ + /* xl_xact_relfilenodes follows if XINFO_HAS_RELFILENODES */ + /* xl_xact_invals follows if XINFO_HAS_INVALS */ + /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ + /* xl_xact_origin follows if XINFO_HAS_ORIGIN, stored unaligned! */ +} xl_xact_commit; + +typedef struct xl_xact_abort +{ + TimestampTz xact_time; /* time of abort */ + + /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ + /* No db_info required */ + /* xl_xact_subxacts follows if HAS_SUBXACT */ + /* xl_xact_relfilenodes follows if HAS_RELFILENODES */ + /* No invalidation messages needed. */ + /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ +} xl_xact_abort; + +static void extractPageInfo(XLogReaderState *record); +static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); + +typedef struct XLogPageReadPrivate +{ + const char *archivedir; + TimeLineID tli; + uint32 xlog_seg_size; + + bool manual_switch; + bool need_switch; + + int xlogfile; + XLogSegNo xlogsegno; + char xlogpath[MAXPGPATH]; + bool xlogexists; + +#ifdef HAVE_LIBZ + gzFile gz_xlogfile; + char gz_xlogpath[MAXPGPATH]; +#endif +} XLogPageReadPrivate; + +/* An argument for a thread function */ +typedef struct +{ + int thread_num; + XLogPageReadPrivate private_data; + + XLogRecPtr startpoint; + XLogRecPtr endpoint; + XLogSegNo endSegNo; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} xlog_thread_arg; + +static int SimpleXLogPageRead(XLogReaderState *xlogreader, + XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, char *readBuf, + TimeLineID *pageTLI); +static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, + const char *archivedir, + TimeLineID tli, uint32 xlog_seg_size, + bool allocate_reader); +static void CleanupXLogPageRead(XLogReaderState *xlogreader); +static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, + int elevel); + +static XLogSegNo nextSegNoToRead = 0; +static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * extractPageMap() worker. + */ +static void * +doExtractPageMap(void *arg) +{ + xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; + XLogPageReadPrivate *private_data; + XLogReaderState *xlogreader; + XLogSegNo nextSegNo = 0; + char *errormsg; + + private_data = &extract_arg->private_data; +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(private_data->xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Start LSN of thread %d: %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + /* Switch WAL segment manually below without using SimpleXLogPageRead() */ + private_data->manual_switch = true; + + do + { + XLogRecord *record; + + if (interrupted) + elog(ERROR, "Interrupted during WAL reading"); + + record = XLogReadRecord(xlogreader, extract_arg->startpoint, &errormsg); + + if (record == NULL) + { + XLogRecPtr errptr; + + /* + * Try to switch to the next WAL segment. Usually + * SimpleXLogPageRead() does it by itself. But here we need to do it + * manually to support threads. + */ + if (private_data->need_switch) + { + private_data->need_switch = false; + + /* Critical section */ + pthread_lock(&wal_segment_mutex); + Assert(nextSegNoToRead); + private_data->xlogsegno = nextSegNoToRead; + nextSegNoToRead++; + pthread_mutex_unlock(&wal_segment_mutex); + + /* We reach the end */ + if (private_data->xlogsegno > extract_arg->endSegNo) + break; + + /* Adjust next record position */ + GetXLogRecPtr(private_data->xlogsegno, 0, + private_data->xlog_seg_size, + extract_arg->startpoint); + /* Skip over the page header */ + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Thread %d switched to LSN %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + continue; + } + + errptr = extract_arg->startpoint ? + extract_arg->startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(WARNING, "could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(WARNING, "could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + + /* + * If we don't have all WAL files from prev backup start_lsn to current + * start_lsn, we won't be able to build page map and PAGE backup will + * be incorrect. Stop it and throw an error. + */ + PrintXLogCorruptionMsg(private_data, ERROR); + } + + extractPageInfo(xlogreader); + + /* continue reading at next record */ + extract_arg->startpoint = InvalidXLogRecPtr; + + GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, + private_data->xlog_seg_size); + } while (nextSegNo <= extract_arg->endSegNo && + xlogreader->EndRecPtr < extract_arg->endpoint); + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + /* Extracting is successful */ + extract_arg->ret = 0; + return NULL; +} + +/* + * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the + * given timeline. Collect data blocks touched by the WAL records into a page map. + * + * If **prev_segno** is true then read all segments up to **endpoint** segment + * minus one. Else read all segments up to **endpoint** segment. + * + * Pagemap extracting is processed using threads. Eeach thread reads single WAL + * file. + */ +void +extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, bool prev_seg, + parray *files) +{ + int i; + int threads_need = 0; + XLogSegNo endSegNo; + bool extract_isok = true; + pthread_t *threads; + xlog_thread_arg *thread_args; + time_t start_time, + end_time; + + elog(LOG, "Compiling pagemap"); + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + + GetXLogSegNo(endpoint, endSegNo, seg_size); + if (prev_seg) + endSegNo--; + + nextSegNoToRead = 0; + time(&start_time); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + thread_args = (xlog_thread_arg *) palloc(sizeof(xlog_thread_arg)*num_threads); + + /* + * Initialize thread args. + * + * Each thread works with its own WAL segment and we need to adjust + * startpoint value for each thread. + */ + for (i = 0; i < num_threads; i++) + { + InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, + seg_size, false); + thread_args[i].thread_num = i; + + thread_args[i].startpoint = startpoint; + thread_args[i].endpoint = endpoint; + thread_args[i].endSegNo = endSegNo; + /* By default there is some error */ + thread_args[i].ret = 1; + + /* Adjust startpoint to the next thread */ + if (nextSegNoToRead == 0) + GetXLogSegNo(startpoint, nextSegNoToRead, seg_size); + + nextSegNoToRead++; + /* + * If we need to read less WAL segments than num_threads, create less + * threads. + */ + if (nextSegNoToRead > endSegNo) + break; + GetXLogRecPtr(nextSegNoToRead, 0, seg_size, startpoint); + /* Skip over the page header */ + startpoint += SizeOfXLogLongPHD; + + threads_need++; + } + + /* Run threads */ + for (i = 0; i < threads_need; i++) + { + elog(VERBOSE, "Start WAL reader thread: %d", i); + pthread_create(&threads[i], NULL, doExtractPageMap, &thread_args[i]); + } + + /* Wait for threads */ + for (i = 0; i < threads_need; i++) + { + pthread_join(threads[i], NULL); + if (thread_args[i].ret == 1) + extract_isok = false; + } + + pfree(threads); + pfree(thread_args); + + time(&end_time); + if (extract_isok) + elog(LOG, "Pagemap compiled, time elapsed %.0f sec", + difftime(end_time, start_time)); + else + elog(ERROR, "Pagemap compiling failed"); +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent state. + */ +static void +validate_backup_wal_from_start_to_stop(pgBackup *backup, + char *backup_xlog_path, TimeLineID tli, + uint32 xlog_seg_size) +{ + XLogRecPtr startpoint = backup->start_lsn; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + bool got_endpoint = false; + + xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, + xlog_seg_size, true); + + while (true) + { + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + /* Got WAL record at stop_lsn */ + if (xlogreader->ReadRecPtr == backup->stop_lsn) + { + got_endpoint = true; + break; + } + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (!got_endpoint) + { + PrintXLogCorruptionMsg(&private, WARNING); + + /* + * If we don't have WAL between start_lsn and stop_lsn, + * the backup is definitely corrupted. Update its status. + */ + backup->status = BACKUP_STATUS_CORRUPT; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "There are not enough WAL records to consistenly restore " + "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", + base36enc(backup->start_time), + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn), + (uint32) (backup->stop_lsn >> 32), + (uint32) (backup->stop_lsn)); + } + + /* clean */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent + * state. And check if we have in archive all files needed to restore the backup + * up to the given recovery target. + */ +void +validate_wal(pgBackup *backup, const char *archivedir, + time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 seg_size) +{ + XLogRecPtr startpoint = backup->start_lsn; + const char *backup_id; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + TransactionId last_xid = InvalidTransactionId; + TimestampTz last_time = 0; + char last_timestamp[100], + target_timestamp[100]; + bool all_wal = false; + char backup_xlog_path[MAXPGPATH]; + + /* We need free() this later */ + backup_id = base36enc(backup->start_time); + + if (!XRecOffIsValid(backup->start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", + (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), + backup_id); + + if (!XRecOffIsValid(backup->stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", + (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), + backup_id); + + /* + * Check that the backup has all wal files needed + * for recovery to consistent state. + */ + if (backup->stream) + { + snprintf(backup_xlog_path, sizeof(backup_xlog_path), "/%s/%s/%s/%s", + backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); + + validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, + seg_size); + } + else + validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli, + seg_size); + + if (backup->status == BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s WAL segments are corrupted", backup_id); + return; + } + /* + * If recovery target is provided check that we can restore backup to a + * recovery target time or xid. + */ + if (!TransactionIdIsValid(target_xid) && target_time == 0 && !XRecOffIsValid(target_lsn)) + { + /* Recovery target is not given so exit */ + elog(INFO, "Backup %s WAL segments are valid", backup_id); + return; + } + + /* + * If recovery target is provided, ensure that archive files exist in + * archive directory. + */ + if (dir_is_empty(archivedir)) + elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); + + /* + * Check if we have in archive all files needed to restore backup + * up to the given recovery target. + * In any case we cannot restore to the point before stop_lsn. + */ + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, + true); + + /* We can restore at least up to the backup end */ + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + last_xid = backup->recovery_xid; + + if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) + || (target_time != 0 && backup->recovery_time >= target_time) + || (XRecOffIsValid(target_lsn) && backup->stop_lsn >= target_lsn)) + all_wal = true; + + startpoint = backup->stop_lsn; + while (true) + { + bool timestamp_record; + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + timestamp_record = getRecordTimestamp(xlogreader, &last_time); + if (XLogRecGetXid(xlogreader) != InvalidTransactionId) + last_xid = XLogRecGetXid(xlogreader); + + /* Check target xid */ + if (TransactionIdIsValid(target_xid) && target_xid == last_xid) + { + all_wal = true; + break; + } + /* Check target time */ + else if (target_time != 0 && timestamp_record && timestamptz_to_time_t(last_time) >= target_time) + { + all_wal = true; + break; + } + /* If there are no target xid and target time */ + else if (!TransactionIdIsValid(target_xid) && target_time == 0 && + xlogreader->ReadRecPtr == backup->stop_lsn) + { + all_wal = true; + /* We don't stop here. We want to get last_xid and last_time */ + } + + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (last_time > 0) + time2iso(last_timestamp, lengthof(last_timestamp), + timestamptz_to_time_t(last_time)); + + /* There are all needed WAL records */ + if (all_wal) + elog(INFO, "backup validation completed successfully on time %s and xid " XID_FMT, + last_timestamp, last_xid); + /* Some needed WAL records are absent */ + else + { + PrintXLogCorruptionMsg(&private, WARNING); + + elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, + last_timestamp, last_xid); + + if (target_time > 0) + time2iso(target_timestamp, lengthof(target_timestamp), + target_time); + if (TransactionIdIsValid(target_xid) && target_time != 0) + elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, + target_timestamp, target_xid); + else if (TransactionIdIsValid(target_xid)) + elog(ERROR, "not enough WAL records to xid " XID_FMT, + target_xid); + else if (target_time != 0) + elog(ERROR, "not enough WAL records to time %s", + target_timestamp); + else if (XRecOffIsValid(target_lsn)) + elog(ERROR, "not enough WAL records to lsn %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + } + + /* clean */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); +} + +/* + * Read from archived WAL segments latest recovery time and xid. All necessary + * segments present at archive folder. We waited **stop_lsn** in + * pg_stop_backup(). + */ +bool +read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr start_lsn, XLogRecPtr stop_lsn, + time_t *recovery_time, TransactionId *recovery_xid) +{ + XLogRecPtr startpoint = stop_lsn; + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + bool res; + + if (!XRecOffIsValid(start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X", + (uint32) (start_lsn >> 32), (uint32) (start_lsn)); + + if (!XRecOffIsValid(stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + + /* Read records from stop_lsn down to start_lsn */ + do + { + XLogRecord *record; + TimestampTz last_time = 0; + char *errormsg; + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + XLogRecPtr errptr; + + errptr = startpoint ? startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(ERROR, "could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(ERROR, "could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + } + + /* Read previous record */ + startpoint = record->xl_prev; + + if (getRecordTimestamp(xlogreader, &last_time)) + { + *recovery_time = timestamptz_to_time_t(last_time); + *recovery_xid = XLogRecGetXid(xlogreader); + + /* Found timestamp in WAL record 'record' */ + res = true; + goto cleanup; + } + } while (startpoint >= start_lsn); + + /* Didn't find timestamp from WAL records between start_lsn and stop_lsn */ + res = false; + +cleanup: + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + +/* + * Check if there is a WAL segment file in 'archivedir' which contains + * 'target_lsn'. + */ +bool +wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, + TimeLineID target_tli, uint32 seg_size) +{ + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + char *errormsg; + bool res; + + if (!XRecOffIsValid(target_lsn)) + elog(ERROR, "Invalid target_lsn value %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + + xlogreader = InitXLogPageRead(&private, archivedir, target_tli, seg_size, + true); + + res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; + /* Didn't find 'target_lsn' and there is no error, return false */ + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf) +{ + int errnum; + const char *errmsg; + + errmsg = gzerror(gzf, &errnum); + if (errnum == Z_ERRNO) + return strerror(errno); + else + return errmsg; +} +#endif + +/* XLogreader callback function, to read a WAL page */ +static int +SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, char *readBuf, + TimeLineID *pageTLI) +{ + XLogPageReadPrivate *private_data; + uint32 targetPageOff; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + targetPageOff = targetPagePtr % private_data->xlog_seg_size; + + /* + * See if we need to switch to a new segment because the requested record + * is not in the currently open one. + */ + if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size)) + { + CleanupXLogPageRead(xlogreader); + /* + * Do not switch to next WAL segment in this function. Currently it is + * manually switched only in doExtractPageMap(). + */ + if (private_data->manual_switch) + { + private_data->need_switch = true; + return -1; + } + } + + GetXLogSegNo(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size); + + /* Try to switch to the next WAL segment */ + if (!private_data->xlogexists) + { + char xlogfname[MAXFNAMELEN]; + + GetXLogFileName(xlogfname, private_data->tli, private_data->xlogsegno, + private_data->xlog_seg_size); + snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", + private_data->archivedir, xlogfname); + + if (fileExists(private_data->xlogpath)) + { + elog(LOG, "Opening WAL segment \"%s\"", private_data->xlogpath); + + private_data->xlogexists = true; + private_data->xlogfile = open(private_data->xlogpath, + O_RDONLY | PG_BINARY, 0); + + if (private_data->xlogfile < 0) + { + elog(WARNING, "Could not open WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + } +#ifdef HAVE_LIBZ + /* Try to open compressed WAL segment */ + else + { + snprintf(private_data->gz_xlogpath, + sizeof(private_data->gz_xlogpath), "%s.gz", + private_data->xlogpath); + if (fileExists(private_data->gz_xlogpath)) + { + elog(LOG, "Opening compressed WAL segment \"%s\"", + private_data->gz_xlogpath); + + private_data->xlogexists = true; + private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, + "rb"); + if (private_data->gz_xlogfile == NULL) + { + elog(WARNING, "Could not open compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, strerror(errno)); + return -1; + } + } + } +#endif + + /* Exit without error if WAL segment doesn't exist */ + if (!private_data->xlogexists) + return -1; + } + + /* + * At this point, we have the right segment open. + */ + Assert(private_data->xlogexists); + + /* Read the requested page */ + if (private_data->xlogfile != -1) + { + if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) + { + elog(WARNING, "Could not seek in WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + + if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Could not read from WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + } +#ifdef HAVE_LIBZ + else + { + if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + { + elog(WARNING, "Could not seek in compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + + if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Could not read from compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + } +#endif + + *pageTLI = private_data->tli; + return XLOG_BLCKSZ; +} + +/* + * Initialize WAL segments reading. + */ +static XLogReaderState * +InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, + TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader) +{ + XLogReaderState *xlogreader = NULL; + + MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); + private_data->archivedir = archivedir; + private_data->tli = tli; + private_data->xlog_seg_size = xlog_seg_size; + private_data->xlogfile = -1; + + if (allocate_reader) + { +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + } + + return xlogreader; +} + +/* + * Cleanup after WAL segment reading. + */ +static void +CleanupXLogPageRead(XLogReaderState *xlogreader) +{ + XLogPageReadPrivate *private_data; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + if (private_data->xlogfile >= 0) + { + close(private_data->xlogfile); + private_data->xlogfile = -1; + } +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + { + gzclose(private_data->gz_xlogfile); + private_data->gz_xlogfile = NULL; + } +#endif + private_data->xlogexists = false; +} + +static void +PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) +{ + if (private_data->xlogpath[0] != 0) + { + /* + * XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status. + */ + if (!private_data->xlogexists) + elog(elevel, "WAL segment \"%s\" is absent", private_data->xlogpath); + else if (private_data->xlogfile != -1) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->xlogpath); +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->gz_xlogpath); +#endif + } +} + +/* + * Extract information about blocks modified in this record. + */ +static void +extractPageInfo(XLogReaderState *record) +{ + uint8 block_id; + RmgrId rmid = XLogRecGetRmid(record); + uint8 info = XLogRecGetInfo(record); + uint8 rminfo = info & ~XLR_INFO_MASK; + + /* Is this a special record type that I recognize? */ + + if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE) + { + /* + * New databases can be safely ignored. They would be completely + * copied if found. + */ + } + else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP) + { + /* + * An existing database was dropped. It is fine to ignore that + * they will be removed appropriately. + */ + } + else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE) + { + /* + * We can safely ignore these. The file will be removed when + * combining the backups in the case of differential on. + */ + } + else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE) + { + /* + * We can safely ignore these. When we compare the sizes later on, + * we'll notice that they differ, and copy the missing tail from + * source system. + */ + } + else if (info & XLR_SPECIAL_REL_UPDATE) + { + /* + * This record type modifies a relation file in some special way, but + * we don't recognize the type. That's bad - we don't know how to + * track that change. + */ + elog(ERROR, "WAL record modifies a relation, but record type is not recognized\n" + "lsn: %X/%X, rmgr: %s, info: %02X", + (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr), + RmgrNames[rmid], info); + } + + for (block_id = 0; block_id <= record->max_block_id; block_id++) + { + RelFileNode rnode; + ForkNumber forknum; + BlockNumber blkno; + + if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) + continue; + + /* We only care about the main fork; others are copied in toto */ + if (forknum != MAIN_FORKNUM) + continue; + + process_block_change(forknum, rnode, blkno); + } +} + +/* + * Extract timestamp from WAL record. + * + * If the record contains a timestamp, returns true, and saves the timestamp + * in *recordXtime. If the record type has no timestamp, returns false. + * Currently, only transaction commit/abort records and restore points contain + * timestamps. + */ +static bool +getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + uint8 xact_info = info & XLOG_XACT_OPMASK; + uint8 rmid = XLogRecGetRmid(record); + + if (rmid == RM_XLOG_ID && info == XLOG_RESTORE_POINT) + { + *recordXtime = ((xl_restore_point *) XLogRecGetData(record))->rp_time; + return true; + } + else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_COMMIT || + xact_info == XLOG_XACT_COMMIT_PREPARED)) + { + *recordXtime = ((xl_xact_commit *) XLogRecGetData(record))->xact_time; + return true; + } + else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_ABORT || + xact_info == XLOG_XACT_ABORT_PREPARED)) + { + *recordXtime = ((xl_xact_abort *) XLogRecGetData(record))->xact_time; + return true; + } + + return false; +} + diff --git a/src/pg_probackup.c b/src/pg_probackup.c new file mode 100644 index 00000000..a39ea5a8 --- /dev/null +++ b/src/pg_probackup.c @@ -0,0 +1,634 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.c: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "streamutil.h" +#include "utils/thread.h" + +#include +#include +#include +#include +#include +#include "pg_getopt.h" + +const char *PROGRAM_VERSION = "2.0.18"; +const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; +const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; + +/* directory options */ +char *backup_path = NULL; +char *pgdata = NULL; +/* + * path or to the data files in the backup catalog + * $BACKUP_PATH/backups/instance_name + */ +char backup_instance_path[MAXPGPATH]; +/* + * path or to the wal files in the backup catalog + * $BACKUP_PATH/wal/instance_name + */ +char arclog_path[MAXPGPATH] = ""; + +/* common options */ +static char *backup_id_string = NULL; +int num_threads = 1; +bool stream_wal = false; +bool progress = false; +#if PG_VERSION_NUM >= 100000 +char *replication_slot = NULL; +#endif + +/* backup options */ +bool backup_logs = false; +bool smooth_checkpoint; +bool is_remote_backup = false; +/* Wait timeout for WAL segment archiving */ +uint32 archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; +const char *master_db = NULL; +const char *master_host = NULL; +const char *master_port= NULL; +const char *master_user = NULL; +uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; + +/* restore options */ +static char *target_time; +static char *target_xid; +static char *target_lsn; +static char *target_inclusive; +static TimeLineID target_tli; +static bool target_immediate; +static char *target_name = NULL; +static char *target_action = NULL; + +static pgRecoveryTarget *recovery_target_options = NULL; + +bool restore_as_replica = false; +bool restore_no_validate = false; + +/* delete options */ +bool delete_wal = false; +bool delete_expired = false; +bool apply_to_all = false; +bool force_delete = false; + +/* retention options */ +uint32 retention_redundancy = 0; +uint32 retention_window = 0; + +/* compression options */ +CompressAlg compress_alg = COMPRESS_ALG_DEFAULT; +int compress_level = COMPRESS_LEVEL_DEFAULT; +bool compress_shortcut = false; + + +/* other options */ +char *instance_name; +uint64 system_identifier = 0; + +/* + * Starting from PostgreSQL 11 WAL segment size may vary. Prior to + * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. + */ +#if PG_VERSION_NUM >= 110000 +uint32 xlog_seg_size = 0; +#else +uint32 xlog_seg_size = XLOG_SEG_SIZE; +#endif + +/* archive push options */ +static char *wal_file_path; +static char *wal_file_name; +static bool file_overwrite = false; + +/* show options */ +ShowFormat show_format = SHOW_PLAIN; + +/* current settings */ +pgBackup current; +ProbackupSubcmd backup_subcmd = NO_CMD; + +static bool help_opt = false; + +static void opt_backup_mode(pgut_option *opt, const char *arg); +static void opt_log_level_console(pgut_option *opt, const char *arg); +static void opt_log_level_file(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); +static void opt_show_format(pgut_option *opt, const char *arg); + +static void compress_init(void); + +static pgut_option options[] = +{ + /* directory options */ + { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, + { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, + { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, + /* common options */ + { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, + { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, + { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, + { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMDLINE }, + /* backup options */ + { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, + { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, + { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, + { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, + { 'u', 11, "archive-timeout", &archive_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + { 'b', 12, "delete-wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 13, "delete-expired", &delete_expired, SOURCE_CMDLINE }, + { 's', 14, "master-db", &master_db, SOURCE_CMDLINE, }, + { 's', 15, "master-host", &master_host, SOURCE_CMDLINE, }, + { 's', 16, "master-port", &master_port, SOURCE_CMDLINE, }, + { 's', 17, "master-user", &master_user, SOURCE_CMDLINE, }, + { 'u', 18, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* TODO not completed feature. Make it unavailiable from user level + { 'b', 18, "remote", &is_remote_backup, SOURCE_CMDLINE, }, */ + /* restore options */ + { 's', 20, "time", &target_time, SOURCE_CMDLINE }, + { 's', 21, "xid", &target_xid, SOURCE_CMDLINE }, + { 's', 22, "inclusive", &target_inclusive, SOURCE_CMDLINE }, + { 'u', 23, "timeline", &target_tli, SOURCE_CMDLINE }, + { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, + { 'b', 24, "immediate", &target_immediate, SOURCE_CMDLINE }, + { 's', 25, "recovery-target-name", &target_name, SOURCE_CMDLINE }, + { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, + { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, + { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, + { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, + /* delete options */ + { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, + { 'b', 132, "all", &apply_to_all, SOURCE_CMDLINE }, + /* TODO not implemented yet */ + { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, + /* retention options */ + { 'u', 134, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, + { 'u', 135, "retention-window", &retention_window, SOURCE_CMDLINE }, + /* compression options */ + { 'f', 136, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 137, "compress-level", &compress_level, SOURCE_CMDLINE }, + { 'b', 138, "compress", &compress_shortcut, SOURCE_CMDLINE }, + /* logging options */ + { 'f', 140, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, + { 'f', 141, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, + { 's', 142, "log-filename", &log_filename, SOURCE_CMDLINE }, + { 's', 143, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, + { 's', 144, "log-directory", &log_directory, SOURCE_CMDLINE }, + { 'u', 145, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, + { 'u', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MIN }, + /* connection options */ + { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, + { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, + { 's', 'p', "pgport", &port, SOURCE_CMDLINE }, + { 's', 'U', "pguser", &username, SOURCE_CMDLINE }, + { 'B', 'w', "no-password", &prompt_password, SOURCE_CMDLINE }, + { 'b', 'W', "password", &force_password, SOURCE_CMDLINE }, + /* other options */ + { 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, + { 's', 151, "instance", &instance_name, SOURCE_CMDLINE }, +#if PG_VERSION_NUM >= 110000 + { 'u', 152, "xlog-seg-size", &xlog_seg_size, SOURCE_FILE_STRICT}, +#endif + /* archive-push options */ + { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, + { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, + { 'b', 162, "overwrite", &file_overwrite, SOURCE_CMDLINE }, + /* show options */ + { 'f', 170, "format", opt_show_format, SOURCE_CMDLINE }, + { 0 } +}; + +/* + * Entry point of pg_probackup command. + */ +int +main(int argc, char *argv[]) +{ + char *command = NULL, + *command_name; + /* Check if backup_path is directory. */ + struct stat stat_buf; + int rc; + + /* initialize configuration */ + pgBackupInit(¤t); + + PROGRAM_NAME = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], "pgscripts"); + +#if PG_VERSION_NUM >= 110000 + /* + * Reset WAL segment size, we will retreive it using RetrieveWalSegSize() + * later. + */ + WalSegSz = 0; +#endif + + /* + * Save main thread's tid. It is used call exit() in case of errors. + */ + main_tid = pthread_self(); + + /* Parse subcommands and non-subcommand options */ + if (argc > 1) + { + if (strcmp(argv[1], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[1], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[1], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[1], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[1], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[1], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[1], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[1], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[1], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "merge") == 0) + backup_subcmd = MERGE_CMD; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[1], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[1], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "--help") == 0 || + strcmp(argv[1], "-?") == 0 || + strcmp(argv[1], "help") == 0) + { + if (argc > 2) + help_command(argv[2]); + else + help_pg_probackup(); + } + else if (strcmp(argv[1], "--version") == 0 + || strcmp(argv[1], "version") == 0 + || strcmp(argv[1], "-V") == 0) + { +#ifdef PGPRO_VERSION + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif + exit(0); + } + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); + } + + if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); + + /* + * Make command string before getopt_long() will call. It permutes the + * content of argv. + */ + command_name = pstrdup(argv[1]); + if (backup_subcmd == BACKUP_CMD || + backup_subcmd == RESTORE_CMD || + backup_subcmd == VALIDATE_CMD || + backup_subcmd == DELETE_CMD || + backup_subcmd == MERGE_CMD) + { + int i, + len = 0, + allocated = 0; + + allocated = sizeof(char) * MAXPGPATH; + command = (char *) palloc(allocated); + + for (i = 0; i < argc; i++) + { + int arglen = strlen(argv[i]); + + if (arglen + len > allocated) + { + allocated *= 2; + command = repalloc(command, allocated); + } + + strncpy(command + len, argv[i], arglen); + len += arglen; + command[len++] = ' '; + } + + command[len] = '\0'; + } + + optind += 1; + /* Parse command line arguments */ + pgut_getopt(argc, argv, options); + + if (help_opt) + help_command(command_name); + + /* backup_path is required for all pg_probackup commands except help */ + if (backup_path == NULL) + { + /* + * If command line argument is not set, try to read BACKUP_PATH + * from environment variable + */ + backup_path = getenv("BACKUP_PATH"); + if (backup_path == NULL) + elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + } + canonicalize_path(backup_path); + + /* Ensure that backup_path is an absolute path */ + if (!is_absolute_path(backup_path)) + elog(ERROR, "-B, --backup-path must be an absolute path"); + + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + + /* command was initialized for a few commands */ + if (command) + { + elog_file(INFO, "command: %s", command); + + pfree(command); + command = NULL; + } + + /* Option --instance is required for all commands except init and show */ + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != VALIDATE_CMD) + { + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + } + + /* + * If --instance option was passed, construct paths for backup data and + * xlog files of this backup instance. + */ + if (instance_name) + { + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + + /* + * Ensure that requested backup instance exists. + * for all commands except init, which doesn't take this parameter + * and add-instance which creates new instance. + */ + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) + { + if (access(backup_instance_path, F_OK) != 0) + elog(ERROR, "Instance '%s' does not exist in this backup catalog", + instance_name); + } + } + + /* + * Read options from env variables or from config file, + * unless we're going to set them via set-config. + */ + if (instance_name && backup_subcmd != SET_CONFIG_CMD) + { + char path[MAXPGPATH]; + + /* Read environment variables */ + pgut_getopt_env(options); + + /* Read options from configuration file */ + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, true); + } + + /* Initialize logger */ + init_logger(backup_path); + + /* + * We have read pgdata path from command line or from configuration file. + * Ensure that pgdata is an absolute path. + */ + if (pgdata != NULL && !is_absolute_path(pgdata)) + elog(ERROR, "-D, --pgdata must be an absolute path"); + +#if PG_VERSION_NUM >= 110000 + /* Check xlog-seg-size option */ + if (instance_name && + backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != ADD_INSTANCE_CMD && !IsValidWalSegSize(xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", xlog_seg_size); +#endif + + /* Sanity check of --backup-id option */ + if (backup_id_string != NULL) + { + if (backup_subcmd != RESTORE_CMD && + backup_subcmd != VALIDATE_CMD && + backup_subcmd != DELETE_CMD && + backup_subcmd != MERGE_CMD && + backup_subcmd != SHOW_CMD) + elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", + command_name); + + current.backup_id = base36dec(backup_id_string); + if (current.backup_id == 0) + elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); + } + + /* Setup stream options. They are used in streamutil.c. */ + if (host != NULL) + dbhost = pstrdup(host); + if (port != NULL) + dbport = pstrdup(port); + if (username != NULL) + dbuser = pstrdup(username); + + /* setup exclusion list for file search */ + if (!backup_logs) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ + + /* Set 'pg_log' in first empty slot */ + pgdata_exclude_dir[i] = "pg_log"; + } + + if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) + { + /* parse all recovery target options into recovery_target_options structure */ + recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, + target_inclusive, target_tli, target_lsn, target_immediate, + target_name, target_action, restore_no_validate); + } + + if (num_threads < 1) + num_threads = 1; + + compress_init(); + + /* do actual operation */ + switch (backup_subcmd) + { + case ARCHIVE_PUSH_CMD: + return do_archive_push(wal_file_path, wal_file_name, file_overwrite); + case ARCHIVE_GET_CMD: + return do_archive_get(wal_file_path, wal_file_name); + case ADD_INSTANCE_CMD: + return do_add_instance(); + case DELETE_INSTANCE_CMD: + return do_delete_instance(); + case INIT_CMD: + return do_init(); + case BACKUP_CMD: + { + const char *backup_mode; + time_t start_time; + + start_time = time(NULL); + backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; + + elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", + PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, + stream_wal ? "true" : "false", is_remote_backup ? "true" : "false"); + + return do_backup(start_time); + } + case RESTORE_CMD: + return do_restore_or_validate(current.backup_id, + recovery_target_options, + true); + case VALIDATE_CMD: + if (current.backup_id == 0 && target_time == 0 && target_xid == 0) + return do_validate_all(); + else + return do_restore_or_validate(current.backup_id, + recovery_target_options, + false); + case SHOW_CMD: + return do_show(current.backup_id); + case DELETE_CMD: + if (delete_expired && backup_id_string) + elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); + if (!delete_expired && !delete_wal && !backup_id_string) + elog(ERROR, "You must specify at least one of the delete options: --expired |--wal |--backup_id"); + if (delete_wal && !delete_expired && !backup_id_string) + return do_retention_purge(); + if (delete_expired) + return do_retention_purge(); + else + return do_delete(current.backup_id); + case MERGE_CMD: + do_merge(current.backup_id); + break; + case SHOW_CONFIG_CMD: + return do_configure(true); + case SET_CONFIG_CMD: + return do_configure(false); + case NO_CMD: + /* Should not happen */ + elog(ERROR, "Unknown subcommand"); + } + + return 0; +} + +static void +opt_backup_mode(pgut_option *opt, const char *arg) +{ + current.backup_mode = parse_backup_mode(arg); +} + +static void +opt_log_level_console(pgut_option *opt, const char *arg) +{ + log_level_console = parse_log_level(arg); +} + +static void +opt_log_level_file(pgut_option *opt, const char *arg) +{ + log_level_file = parse_log_level(arg); +} + +static void +opt_show_format(pgut_option *opt, const char *arg) +{ + const char *v = arg; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0) + { + if (pg_strncasecmp("plain", v, len) == 0) + show_format = SHOW_PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + show_format = SHOW_JSON; + else + elog(ERROR, "Invalid show format \"%s\"", arg); + } + else + elog(ERROR, "Invalid show format \"%s\"", arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + compress_alg = parse_compress_alg(arg); +} + +/* + * Initialize compress and sanity checks for compress. + */ +static void +compress_init(void) +{ + /* Default algorithm is zlib */ + if (compress_shortcut) + compress_alg = ZLIB_COMPRESS; + + if (backup_subcmd != SET_CONFIG_CMD) + { + if (compress_level != COMPRESS_LEVEL_DEFAULT + && compress_alg == NOT_DEFINED_COMPRESS) + elog(ERROR, "Cannot specify compress-level option without compress-alg option"); + } + + if (compress_level < 0 || compress_level > 9) + elog(ERROR, "--compress-level value must be in the range from 0 to 9"); + + if (compress_level == 0) + compress_alg = NOT_DEFINED_COMPRESS; + + if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) + { +#ifndef HAVE_LIBZ + if (compress_alg == ZLIB_COMPRESS) + elog(ERROR, "This build does not support zlib compression"); + else +#endif + if (compress_alg == PGLZ_COMPRESS && num_threads > 1) + elog(ERROR, "Multithread backup does not support pglz compression"); + } +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h new file mode 100644 index 00000000..8f3a0fea --- /dev/null +++ b/src/pg_probackup.h @@ -0,0 +1,620 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.h: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef PG_PROBACKUP_H +#define PG_PROBACKUP_H + +#include "postgres_fe.h" + +#include +#include + +#include "access/timeline.h" +#include "access/xlogdefs.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum.h" +#include "utils/pg_crc.h" +#include "common/relpath.h" +#include "port.h" + +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif + +#include "utils/parray.h" +#include "utils/pgut.h" + +#include "datapagemap.h" + +# define PG_STOP_BACKUP_TIMEOUT 300 +/* + * Macro needed to parse ptrack. + * NOTE Keep those values syncronised with definitions in ptrack.h + */ +#define PTRACK_BITS_PER_HEAPBLOCK 1 +#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) + +/* Directory/File names */ +#define DATABASE_DIR "database" +#define BACKUPS_DIR "backups" +#if PG_VERSION_NUM >= 100000 +#define PG_XLOG_DIR "pg_wal" +#else +#define PG_XLOG_DIR "pg_xlog" +#endif +#define PG_TBLSPC_DIR "pg_tblspc" +#define PG_GLOBAL_DIR "global" +#define BACKUP_CONTROL_FILE "backup.control" +#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf" +#define BACKUP_CATALOG_PID "pg_probackup.pid" +#define DATABASE_FILE_LIST "backup_content.control" +#define PG_BACKUP_LABEL_FILE "backup_label" +#define PG_BLACK_LIST "black_list" +#define PG_TABLESPACE_MAP_FILE "tablespace_map" + +#define LOG_FILENAME_DEFAULT "pg_probackup.log" +#define LOG_DIRECTORY_DEFAULT "log" +/* Direcotry/File permission */ +#define DIR_PERMISSION (0700) +#define FILE_PERMISSION (0600) + +/* 64-bit xid support for PGPRO_EE */ +#ifndef PGPRO_EE +#define XID_FMT "%u" +#endif + +typedef enum CompressAlg +{ + NOT_DEFINED_COMPRESS = 0, + NONE_COMPRESS, + PGLZ_COMPRESS, + ZLIB_COMPRESS, +} CompressAlg; + +/* Information about single file (or dir) in backup */ +typedef struct pgFile +{ + char *name; /* file or directory name */ + mode_t mode; /* protection (file type and permission) */ + size_t size; /* size of the file */ + size_t read_size; /* size of the portion read (if only some pages are + backed up, it's different from size) */ + int64 write_size; /* size of the backed-up file. BYTES_INVALID means + that the file existed but was not backed up + because not modified since last backup. */ + /* we need int64 here to store '-1' value */ + pg_crc32 crc; /* CRC value of the file, regular file only */ + char *linked; /* path of the linked file */ + bool is_datafile; /* true if the file is PostgreSQL data file */ + char *path; /* absolute path of the file */ + Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ + Oid dbOid; /* dbOid extracted from path, if applicable */ + Oid relOid; /* relOid extracted from path, if applicable */ + char *forkName; /* forkName extracted from path, if applicable */ + int segno; /* Segment number for ptrack */ + int n_blocks; /* size of the file in blocks, readed during DELTA backup */ + bool is_cfs; /* Flag to distinguish files compressed by CFS*/ + bool is_database; + bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ + CompressAlg compress_alg; /* compression algorithm applied to the file */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ + datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, + * i.e. datafiles without _ptrack */ +} pgFile; + +/* Special values of datapagemap_t bitmapsize */ +#define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ + +/* Current state of backup */ +typedef enum BackupStatus +{ + BACKUP_STATUS_INVALID, /* the pgBackup is invalid */ + BACKUP_STATUS_OK, /* completed backup */ + BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ + BACKUP_STATUS_RUNNING, /* running backup */ + BACKUP_STATUS_MERGING, /* merging backups */ + BACKUP_STATUS_DELETING, /* data files are being deleted */ + BACKUP_STATUS_DELETED, /* data files have been deleted */ + BACKUP_STATUS_DONE, /* completed but not validated yet */ + BACKUP_STATUS_ORPHAN, /* backup validity is unknown but at least one parent backup is corrupted */ + BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ +} BackupStatus; + +typedef enum BackupMode +{ + BACKUP_MODE_INVALID = 0, + BACKUP_MODE_DIFF_PAGE, /* incremental page backup */ + BACKUP_MODE_DIFF_PTRACK, /* incremental page backup with ptrack system */ + BACKUP_MODE_DIFF_DELTA, /* incremental page backup with lsn comparison */ + BACKUP_MODE_FULL /* full backup */ +} BackupMode; + +typedef enum ProbackupSubcmd +{ + NO_CMD = 0, + INIT_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + DELETE_CMD, + MERGE_CMD, + SHOW_CMD, + SET_CONFIG_CMD, + SHOW_CONFIG_CMD +} ProbackupSubcmd; + +typedef enum ShowFormat +{ + SHOW_PLAIN, + SHOW_JSON +} ShowFormat; + + +/* special values of pgBackup fields */ +#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ +#define BYTES_INVALID (-1) +#define BLOCKNUM_INVALID (-1) + +typedef struct pgBackupConfig +{ + uint64 system_identifier; + uint32 xlog_seg_size; + + char *pgdata; + const char *pgdatabase; + const char *pghost; + const char *pgport; + const char *pguser; + + const char *master_host; + const char *master_port; + const char *master_db; + const char *master_user; + int replica_timeout; + + int archive_timeout; + + int log_level_console; + int log_level_file; + char *log_filename; + char *error_log_filename; + char *log_directory; + int log_rotation_size; + int log_rotation_age; + + uint32 retention_redundancy; + uint32 retention_window; + + CompressAlg compress_alg; + int compress_level; +} pgBackupConfig; + + +/* Information about single backup stored in backup.conf */ + + +typedef struct pgBackup pgBackup; + +struct pgBackup +{ + BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/ + time_t backup_id; /* Identifier of the backup. + * Currently it's the same as start_time */ + BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ + TimeLineID tli; /* timeline of start and stop baskup lsns */ + XLogRecPtr start_lsn; /* backup's starting transaction log location */ + XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ + time_t start_time; /* since this moment backup has status + * BACKUP_STATUS_RUNNING */ + time_t end_time; /* the moment when backup was finished, or the moment + * when we realized that backup is broken */ + time_t recovery_time; /* Earliest moment for which you can restore + * the state of the database cluster using + * this backup */ + TransactionId recovery_xid; /* Earliest xid for which you can restore + * the state of the database cluster using + * this backup */ + /* + * Amount of raw data. For a full backup, this is the total amount of + * data while for a differential backup this is just the difference + * of data taken. + * BYTES_INVALID means nothing was backed up. + */ + int64 data_bytes; + /* Size of WAL files in archive needed to restore this backup */ + int64 wal_bytes; + + CompressAlg compress_alg; + int compress_level; + + /* Fields needed for compatibility check */ + uint32 block_size; + uint32 wal_block_size; + uint32 checksum_version; + + char program_version[100]; + char server_version[100]; + + bool stream; /* Was this backup taken in stream mode? + * i.e. does it include all needed WAL files? */ + bool from_replica; /* Was this backup taken from replica */ + time_t parent_backup; /* Identifier of the previous backup. + * Which is basic backup for this + * incremental backup. */ + pgBackup *parent_backup_link; + char *primary_conninfo; /* Connection parameters of the backup + * in the format suitable for recovery.conf */ +}; + +/* Recovery target for restore and validate subcommands */ +typedef struct pgRecoveryTarget +{ + bool time_specified; + time_t recovery_target_time; + /* add one more field in order to avoid deparsing recovery_target_time back */ + const char *target_time_string; + bool xid_specified; + TransactionId recovery_target_xid; + /* add one more field in order to avoid deparsing recovery_target_xid back */ + const char *target_xid_string; + bool lsn_specified; + XLogRecPtr recovery_target_lsn; + /* add one more field in order to avoid deparsing recovery_target_lsn back */ + const char *target_lsn_string; + TimeLineID recovery_target_tli; + bool recovery_target_inclusive; + bool inclusive_specified; + bool recovery_target_immediate; + const char *recovery_target_name; + const char *recovery_target_action; + bool restore_no_validate; +} pgRecoveryTarget; + +/* Union to ease operations on relation pages */ +typedef union DataPage +{ + PageHeaderData page_data; + char data[BLCKSZ]; +} DataPage; + +typedef struct +{ + const char *from_root; + const char *to_root; + + parray *files_list; + parray *prev_filelist; + XLogRecPtr prev_start_lsn; + + PGconn *backup_conn; + PGcancel *cancel_conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} backup_files_arg; + +/* + * return pointer that exceeds the length of prefix from character string. + * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz". + */ +#define GetRelativePath(str, prefix) \ + ((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1) + +/* + * Return timeline, xlog ID and record offset from an LSN of the type + * 0/B000188, usual result from pg_stop_backup() and friends. + */ +#define XLogDataFromLSN(data, xlogid, xrecoff) \ + sscanf(data, "%X/%X", xlogid, xrecoff) + +#define IsCompressedXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) + +#if PG_VERSION_NUM >= 110000 +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) +#else +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo) +#endif + +/* directory options */ +extern char *backup_path; +extern char backup_instance_path[MAXPGPATH]; +extern char *pgdata; +extern char arclog_path[MAXPGPATH]; + +/* common options */ +extern int num_threads; +extern bool stream_wal; +extern bool progress; +#if PG_VERSION_NUM >= 100000 +/* In pre-10 'replication_slot' is defined in receivelog.h */ +extern char *replication_slot; +#endif + +/* backup options */ +extern bool smooth_checkpoint; +#define ARCHIVE_TIMEOUT_DEFAULT 300 +extern uint32 archive_timeout; +extern bool is_remote_backup; +extern const char *master_db; +extern const char *master_host; +extern const char *master_port; +extern const char *master_user; +#define REPLICA_TIMEOUT_DEFAULT 300 +extern uint32 replica_timeout; + +extern bool is_ptrack_support; +extern bool is_checksum_enabled; +extern bool exclusive_backup; + +/* restore options */ +extern bool restore_as_replica; + +/* delete options */ +extern bool delete_wal; +extern bool delete_expired; +extern bool apply_to_all; +extern bool force_delete; + +/* retention options. 0 disables the option */ +#define RETENTION_REDUNDANCY_DEFAULT 0 +#define RETENTION_WINDOW_DEFAULT 0 + +extern uint32 retention_redundancy; +extern uint32 retention_window; + +/* compression options */ +extern CompressAlg compress_alg; +extern int compress_level; +extern bool compress_shortcut; + +#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS +#define COMPRESS_LEVEL_DEFAULT 1 + +extern CompressAlg parse_compress_alg(const char *arg); +extern const char* deparse_compress_alg(int alg); +/* other options */ +extern char *instance_name; +extern uint64 system_identifier; +extern uint32 xlog_seg_size; + +/* show options */ +extern ShowFormat show_format; + +/* current settings */ +extern pgBackup current; +extern ProbackupSubcmd backup_subcmd; + +/* in dir.c */ +/* exclude directory list for $PGDATA file listing */ +extern const char *pgdata_exclude_dir[]; + +/* in backup.c */ +extern int do_backup(time_t start_time); +extern BackupMode parse_backup_mode(const char *value); +extern const char *deparse_backup_mode(BackupMode mode); +extern void process_block_change(ForkNumber forknum, RelFileNode rnode, + BlockNumber blkno); + +extern char *pg_ptrack_get_block(backup_files_arg *arguments, + Oid dbOid, Oid tblsOid, Oid relOid, + BlockNumber blknum, + size_t *result_size); +/* in restore.c */ +extern int do_restore_or_validate(time_t target_backup_id, + pgRecoveryTarget *rt, + bool is_restore); +extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); +extern bool satisfy_recovery_target(const pgBackup *backup, + const pgRecoveryTarget *rt); +extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); +extern pgRecoveryTarget *parseRecoveryTargetOptions( + const char *target_time, const char *target_xid, + const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, + bool target_immediate, const char *target_name, + const char *target_action, bool restore_no_validate); + +/* in merge.c */ +extern void do_merge(time_t backup_id); + +/* in init.c */ +extern int do_init(void); +extern int do_add_instance(void); + +/* in archive.c */ +extern int do_archive_push(char *wal_file_path, char *wal_file_name, + bool overwrite); +extern int do_archive_get(char *wal_file_path, char *wal_file_name); + + +/* in configure.c */ +extern int do_configure(bool show_only); +extern void pgBackupConfigInit(pgBackupConfig *config); +extern void writeBackupCatalogConfig(FILE *out, pgBackupConfig *config); +extern void writeBackupCatalogConfigFile(pgBackupConfig *config); +extern pgBackupConfig* readBackupCatalogConfigFile(void); + +extern uint32 get_config_xlog_seg_size(void); + +/* in show.c */ +extern int do_show(time_t requested_backup_id); + +/* in delete.c */ +extern int do_delete(time_t backup_id); +extern int do_retention_purge(void); +extern int do_delete_instance(void); + +/* in fetch.c */ +extern char *slurpFile(const char *datadir, + const char *path, + size_t *filesize, + bool safe); +extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); + +/* in help.c */ +extern void help_pg_probackup(void); +extern void help_command(char *command); + +/* in validate.c */ +extern void pgBackupValidate(pgBackup* backup); +extern int do_validate_all(void); + +/* in catalog.c */ +extern pgBackup *read_backup(time_t timestamp); +extern const char *pgBackupGetBackupMode(pgBackup *backup); + +extern parray *catalog_get_backup_list(time_t requested_backup_id); +extern pgBackup *catalog_get_last_data_backup(parray *backup_list, + TimeLineID tli); +extern void catalog_lock(void); +extern void pgBackupWriteControl(FILE *out, pgBackup *backup); +extern void pgBackupWriteBackupControlFile(pgBackup *backup); +extern void pgBackupWriteFileList(pgBackup *backup, parray *files, + const char *root); + +extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); +extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2); +extern int pgBackupCreateDir(pgBackup *backup); +extern void pgBackupInit(pgBackup *backup); +extern void pgBackupCopy(pgBackup *dst, pgBackup *src); +extern void pgBackupFree(void *backup); +extern int pgBackupCompareId(const void *f1, const void *f2); +extern int pgBackupCompareIdDesc(const void *f1, const void *f2); + +extern pgBackup* find_parent_backup(pgBackup *current_backup); + +/* in dir.c */ +extern void dir_list_file(parray *files, const char *root, bool exclude, + bool omit_symlink, bool add_root); +extern void create_data_directories(const char *data_dir, + const char *backup_dir, + bool extract_tablespaces); + +extern void read_tablespace_map(parray *files, const char *backup_dir); +extern void opt_tablespace_map(pgut_option *opt, const char *arg); +extern void check_tablespace_mapping(pgBackup *backup); + +extern void print_file_list(FILE *out, const parray *files, const char *root); +extern parray *dir_read_file_list(const char *root, const char *file_txt); + +extern int dir_create_dir(const char *path, mode_t mode); +extern bool dir_is_empty(const char *path); + +extern bool fileExists(const char *path); +extern size_t pgFileSize(const char *path); + +extern pgFile *pgFileNew(const char *path, bool omit_symlink); +extern pgFile *pgFileInit(const char *path); +extern void pgFileDelete(pgFile *file); +extern void pgFileFree(void *file); +extern pg_crc32 pgFileGetCRC(const char *file_path); +extern int pgFileComparePath(const void *f1, const void *f2); +extern int pgFileComparePathDesc(const void *f1, const void *f2); +extern int pgFileCompareLinked(const void *f1, const void *f2); +extern int pgFileCompareSize(const void *f1, const void *f2); + +/* in data.c */ +extern bool backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, + BackupMode backup_mode, + CompressAlg calg, int clevel); +extern void restore_data_file(const char *to_path, + pgFile *file, bool allow_truncate, + bool write_header); +extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern void move_file(const char *from_root, const char *to_root, pgFile *file); +extern void push_wal_file(const char *from_path, const char *to_path, + bool is_compress, bool overwrite); +extern void get_wal_file(const char *from_path, const char *to_path); + +extern bool calc_file_checksum(pgFile *file); + +/* parsexlog.c */ +extern void extractPageMap(const char *datadir, + TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, + bool prev_seg, parray *backup_files_list); +extern void validate_wal(pgBackup *backup, + const char *archivedir, + time_t target_time, + TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 seg_size); +extern bool read_recovery_info(const char *archivedir, TimeLineID tli, + uint32 seg_size, + XLogRecPtr start_lsn, XLogRecPtr stop_lsn, + time_t *recovery_time, + TransactionId *recovery_xid); +extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, + TimeLineID target_tli, uint32 seg_size); + +/* in util.c */ +extern TimeLineID get_current_timeline(bool safe); +extern void sanityChecks(void); +extern void time2iso(char *buf, size_t len, time_t time); +extern const char *status2str(BackupStatus status); +extern void remove_trailing_space(char *buf, int comment_mark); +extern void remove_not_digit(char *buf, size_t len, const char *str); +extern uint32 get_data_checksum_version(bool safe); +extern const char *base36enc(long unsigned int value); +extern char *base36enc_dup(long unsigned int value); +extern long unsigned int base36dec(const char *text); +extern uint64 get_system_identifier(char *pgdata); +extern uint64 get_remote_system_identifier(PGconn *conn); +extern uint32 get_xlog_seg_size(char *pgdata_path); +extern pg_time_t timestamptz_to_time_t(TimestampTz t); +extern int parse_server_version(char *server_version_str); + +/* in status.c */ +extern bool is_pg_running(void); + +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#define mkdir(dir,mode) _mkdir(dir) +#endif +#endif + +#endif /* PG_PROBACKUP_H */ diff --git a/src/restore.c b/src/restore.c new file mode 100644 index 00000000..acd794c8 --- /dev/null +++ b/src/restore.c @@ -0,0 +1,920 @@ +/*------------------------------------------------------------------------- + * + * restore.c: restore DB cluster and archived WAL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include + +#include "catalog/pg_control.h" +#include "utils/logger.h" +#include "utils/thread.h" + +typedef struct +{ + parray *files; + pgBackup *backup; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} restore_files_arg; + +static void restore_backup(pgBackup *backup); +static void create_recovery_conf(time_t backup_id, + pgRecoveryTarget *rt, + pgBackup *backup); +static void *restore_files(void *arg); +static void remove_deleted_files(pgBackup *backup); + + +/* + * Entry point of pg_probackup RESTORE and VALIDATE subcommands. + */ +int +do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, + bool is_restore) +{ + int i = 0; + parray *backups; + pgBackup *current_backup = NULL; + pgBackup *dest_backup = NULL; + pgBackup *base_full_backup = NULL; + pgBackup *corrupted_backup = NULL; + int dest_backup_index = 0; + int base_full_backup_index = 0; + int corrupted_backup_index = 0; + char *action = is_restore ? "Restore":"Validate"; + + if (is_restore) + { + if (pgdata == NULL) + elog(ERROR, + "required parameter not specified: PGDATA (-D, --pgdata)"); + /* Check if restore destination empty */ + if (!dir_is_empty(pgdata)) + elog(ERROR, "restore destination is not empty: \"%s\"", pgdata); + } + + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + + elog(LOG, "%s begin.", action); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find backup range we should restore or validate. */ + while ((i < parray_num(backups)) && !dest_backup) + { + current_backup = (pgBackup *) parray_get(backups, i); + i++; + + /* Skip all backups which started after target backup */ + if (target_backup_id && current_backup->start_time > target_backup_id) + continue; + + /* + * [PGPRO-1164] If BACKUP_ID is not provided for restore command, + * we must find the first valid(!) backup. + */ + + if (is_restore && + target_backup_id == INVALID_BACKUP_ID && + current_backup->status != BACKUP_STATUS_OK) + { + elog(WARNING, "Skipping backup %s, because it has non-valid status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + continue; + } + + /* + * We found target backup. Check its status and + * ensure that it satisfies recovery target. + */ + if ((target_backup_id == current_backup->start_time + || target_backup_id == INVALID_BACKUP_ID)) + { + + /* backup is not ok, + * but in case of CORRUPT, ORPHAN or DONE revalidation can be done, + * in other cases throw an error. + */ + if (current_backup->status != BACKUP_STATUS_OK) + { + if (current_backup->status == BACKUP_STATUS_DONE || + current_backup->status == BACKUP_STATUS_ORPHAN || + current_backup->status == BACKUP_STATUS_CORRUPT) + elog(WARNING, "Backup %s has status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + else + elog(ERROR, "Backup %s has status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + } + + if (rt->recovery_target_tli) + { + parray *timelines; + + elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); + /* Read timeline history files from archives */ + timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); + + if (!satisfy_timeline(timelines, current_backup)) + { + if (target_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "target backup %s does not satisfy target timeline", + base36enc(target_backup_id)); + else + /* Try to find another backup that satisfies target timeline */ + continue; + } + } + + if (!satisfy_recovery_target(current_backup, rt)) + { + if (target_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "target backup %s does not satisfy restore options", + base36enc(target_backup_id)); + else + /* Try to find another backup that satisfies target options */ + continue; + } + + /* + * Backup is fine and satisfies all recovery options. + * Save it as dest_backup + */ + dest_backup = current_backup; + dest_backup_index = i-1; + } + } + + if (dest_backup == NULL) + elog(ERROR, "Backup satisfying target options is not found."); + + /* If we already found dest_backup, look for full backup. */ + if (dest_backup) + { + base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* + * We have found full backup by link, + * now we need to walk the list to find its index. + * + * TODO I think we should rewrite it someday to use double linked list + * and avoid relying on sort order anymore. + */ + for (i = dest_backup_index; i < parray_num(backups); i++) + { + pgBackup * temp_backup = (pgBackup *) parray_get(backups, i); + if (temp_backup->start_time == base_full_backup->start_time) + { + base_full_backup_index = i; + break; + } + } + } + + if (base_full_backup == NULL) + elog(ERROR, "Full backup satisfying target options is not found."); + + /* + * Ensure that directories provided in tablespace mapping are valid + * i.e. empty or not exist. + */ + if (is_restore) + check_tablespace_mapping(dest_backup); + + if (!is_restore || !rt->restore_no_validate) + { + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); + + /* + * Validate backups from base_full_backup to dest_backup. + */ + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + pgBackupValidate(backup); + /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ + if (backup->status == BACKUP_STATUS_CORRUPT) + { + corrupted_backup = backup; + corrupted_backup_index = i; + break; + } + /* We do not validate WAL files of intermediate backups + * It`s done to speed up restore + */ + } + /* There is no point in wal validation + * if there is corrupted backup between base_backup and dest_backup + */ + if (!corrupted_backup) + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->recovery_target_time, + rt->recovery_target_xid, rt->recovery_target_lsn, + base_full_backup->tli, xlog_seg_size); + + /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (corrupted_backup) + { + for (i = corrupted_backup_index - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + /* Mark incremental OK backup as orphan */ + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + char *backup_id, + *corrupted_backup_id; + + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); + + backup_id = base36enc_dup(backup->start_time); + corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); + + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + backup_id, corrupted_backup_id); + + free(backup_id); + free(corrupted_backup_id); + } + } + } + } + + /* + * If dest backup is corrupted or was orphaned in previous check + * produce corresponding error message + */ + if (dest_backup->status == BACKUP_STATUS_OK) + { + if (rt->restore_no_validate) + elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + else + elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + } + else if (dest_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + else if (dest_backup->status == BACKUP_STATUS_ORPHAN) + elog(ERROR, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + else + elog(ERROR, "Backup %s has status: %s", + base36enc(dest_backup->start_time), status2str(dest_backup->status)); + + /* We ensured that all backups are valid, now restore if required */ + if (is_restore) + { + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (rt->lsn_specified && parse_server_version(backup->server_version) < 100000) + elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", + base36enc(dest_backup->start_time), dest_backup->server_version); + + restore_backup(backup); + } + + /* + * Delete files which are not in dest backup file list. Files which were + * deleted between previous and current backup are not in the list. + */ + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + remove_deleted_files(dest_backup); + + /* Create recovery.conf with given recovery target parameters */ + create_recovery_conf(target_backup_id, rt, dest_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(INFO, "%s of backup %s completed.", + action, base36enc(dest_backup->start_time)); + return 0; +} + +/* + * Restore one backup. + */ +void +restore_backup(pgBackup *backup) +{ + char timestamp[100]; + char this_backup_path[MAXPGPATH]; + char database_path[MAXPGPATH]; + char list_path[MAXPGPATH]; + parray *files; + int i; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + restore_files_arg *threads_args; + bool restore_isok = true; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); + + /* confirm block size compatibility */ + if (backup->block_size != BLCKSZ) + elog(ERROR, + "BLCKSZ(%d) is not compatible(%d expected)", + backup->block_size, BLCKSZ); + if (backup->wal_block_size != XLOG_BLCKSZ) + elog(ERROR, + "XLOG_BLCKSZ(%d) is not compatible(%d expected)", + backup->wal_block_size, XLOG_BLCKSZ); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + elog(LOG, "restoring database from backup %s", timestamp); + + /* + * Restore backup directories. + * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id + */ + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + create_data_directories(pgdata, this_backup_path, true); + + /* + * Get list of files which need to be restored. + */ + pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); + pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); + files = dir_read_file_list(database_path, list_path); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_clear_flag(&file->lock); + } + + /* Restore files into target directory */ + for (i = 0; i < num_threads; i++) + { + restore_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->backup = backup; + /* By default there are some error */ + threads_args[i].ret = 1; + + elog(LOG, "Start thread for num:%li", parray_num(files)); + + pthread_create(&threads[i], NULL, restore_files, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + restore_isok = false; + } + if (!restore_isok) + elog(ERROR, "Data files restoring failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); +} + +/* + * Delete files which are not in backup's file list from target pgdata. + * It is necessary to restore incremental backup correctly. + * Files which were deleted between previous and current backup + * are not in the backup's filelist. + */ +static void +remove_deleted_files(pgBackup *backup) +{ + parray *files; + parray *files_restored; + char filelist_path[MAXPGPATH]; + int i; + + pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); + /* Read backup's filelist using target database path as base path */ + files = dir_read_file_list(pgdata, filelist_path); + parray_qsort(files, pgFileComparePathDesc); + + /* Get list of files actually existing in target database */ + files_restored = parray_new(); + dir_list_file(files_restored, pgdata, true, true, false); + /* To delete from leaf, sort in reversed order */ + parray_qsort(files_restored, pgFileComparePathDesc); + + for (i = 0; i < parray_num(files_restored); i++) + { + pgFile *file = (pgFile *) parray_get(files_restored, i); + + /* If the file is not in the file list, delete it */ + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); + } + } + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + parray_walk(files_restored, pgFileFree); + parray_free(files_restored); +} + +/* + * Restore files into $PGDATA. + */ +static void * +restore_files(void *arg) +{ + int i; + restore_files_arg *arguments = (restore_files_arg *)arg; + + for (i = 0; i < parray_num(arguments->files); i++) + { + char from_root[MAXPGPATH]; + char *rel_path; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + pgBackupGetPath(arguments->backup, from_root, + lengthof(from_root), DATABASE_DIR); + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during restore database"); + + rel_path = GetRelativePath(file->path,from_root); + + if (progress) + elog(LOG, "Progress: (%d/%lu). Process file %s ", + i + 1, (unsigned long) parray_num(arguments->files), rel_path); + + /* + * For PAGE and PTRACK backups skip files which haven't changed + * since previous backup and thus were not backed up. + * We cannot do the same when restoring DELTA backup because we need information + * about every file to correctly truncate them. + */ + if (file->write_size == BYTES_INVALID && + (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE + || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) + { + elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + { + elog(VERBOSE, "directory, skip"); + continue; + } + + /* Do not restore tablespace_map file */ + if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, rel_path)) + { + elog(VERBOSE, "skip tablespace_map"); + continue; + } + + /* + * restore the file. + * We treat datafiles separately, cause they were backed up block by + * block and have BackupPageHeader meta information, so we cannot just + * copy the file from backup. + */ + elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", + file->path, file->is_datafile?1:0, file->is_cfs?1:0); + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, pgdata, + file->path + strlen(from_root) + 1); + restore_data_file(to_path, file, + arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + } + else + copy_file(from_root, pgdata, file); + + /* print size of restored file */ + if (file->write_size != BYTES_INVALID) + elog(LOG, "Restored file %s : " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files restoring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* Create recovery.conf with given recovery target parameters */ +static void +create_recovery_conf(time_t backup_id, + pgRecoveryTarget *rt, + pgBackup *backup) +{ + char path[MAXPGPATH]; + FILE *fp; + bool need_restore_conf = false; + + if (!backup->stream + || (rt->time_specified || rt->xid_specified)) + need_restore_conf = true; + + /* No need to generate recovery.conf at all. */ + if (!(need_restore_conf || restore_as_replica)) + return; + + elog(LOG, "----------------------------------------"); + elog(LOG, "creating recovery.conf"); + + snprintf(path, lengthof(path), "%s/recovery.conf", pgdata); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, + strerror(errno)); + + fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", + PROGRAM_VERSION); + + if (need_restore_conf) + { + + fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " + "--wal-file-path %%p --wal-file-name %%f'\n", + PROGRAM_NAME, backup_path, instance_name); + + /* + * We've already checked that only one of the four following mutually + * exclusive options is specified, so the order of calls is insignificant. + */ + if (rt->recovery_target_name) + fprintf(fp, "recovery_target_name = '%s'\n", rt->recovery_target_name); + + if (rt->time_specified) + fprintf(fp, "recovery_target_time = '%s'\n", rt->target_time_string); + + if (rt->xid_specified) + fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); + + if (rt->recovery_target_lsn) + fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); + + if (rt->recovery_target_immediate) + fprintf(fp, "recovery_target = 'immediate'\n"); + + if (rt->inclusive_specified) + fprintf(fp, "recovery_target_inclusive = '%s'\n", + rt->recovery_target_inclusive?"true":"false"); + + if (rt->recovery_target_tli) + fprintf(fp, "recovery_target_timeline = '%u'\n", rt->recovery_target_tli); + + if (rt->recovery_target_action) + fprintf(fp, "recovery_target_action = '%s'\n", rt->recovery_target_action); + } + + if (restore_as_replica) + { + fprintf(fp, "standby_mode = 'on'\n"); + + if (backup->primary_conninfo) + fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + } + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write recovery.conf \"%s\": %s", path, + strerror(errno)); +} + +/* + * Try to read a timeline's history file. + * + * If successful, return the list of component TLIs (the ancestor + * timelines followed by target timeline). If we cannot find the history file, + * assume that the timeline has no parents, and return a list of just the + * specified timeline ID. + * based on readTimeLineHistory() in timeline.c + */ +parray * +readTimeLineHistory_probackup(TimeLineID targetTLI) +{ + parray *result; + char path[MAXPGPATH]; + char fline[MAXPGPATH]; + FILE *fd = NULL; + TimeLineHistoryEntry *entry; + TimeLineHistoryEntry *last_timeline = NULL; + + /* Look for timeline history file in archlog_path */ + snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, + targetTLI); + + /* Timeline 1 does not have a history file */ + if (targetTLI != 1) + { + fd = fopen(path, "rt"); + if (fd == NULL) + { + if (errno != ENOENT) + elog(ERROR, "could not open file \"%s\": %s", path, + strerror(errno)); + + /* There is no history file for target timeline */ + elog(ERROR, "recovery target timeline %u does not exist", + targetTLI); + } + } + + result = parray_new(); + + /* + * Parse the file... + */ + while (fd && fgets(fline, sizeof(fline), fd) != NULL) + { + char *ptr; + TimeLineID tli; + uint32 switchpoint_hi; + uint32 switchpoint_lo; + int nfields; + + for (ptr = fline; *ptr; ptr++) + { + if (!isspace((unsigned char) *ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + nfields = sscanf(fline, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); + + if (nfields < 1) + { + /* expect a numeric timeline ID as first field of line */ + elog(ERROR, + "syntax error in history file: %s. Expected a numeric timeline ID.", + fline); + } + if (nfields != 3) + elog(ERROR, + "syntax error in history file: %s. Expected a transaction log switchpoint location.", + fline); + + if (last_timeline && tli <= last_timeline->tli) + elog(ERROR, + "Timeline IDs must be in increasing sequence."); + + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = tli; + entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; + + last_timeline = entry; + /* Build list with newest item first */ + parray_insert(result, 0, entry); + + /* we ignore the remainder of each line */ + } + + if (fd) + fclose(fd); + + if (last_timeline && targetTLI <= last_timeline->tli) + elog(ERROR, "Timeline IDs must be less than child timeline's ID."); + + /* append target timeline */ + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = targetTLI; + /* LSN in target timeline is valid */ + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; + parray_insert(result, 0, entry); + + return result; +} + +bool +satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) +{ + if (rt->xid_specified) + return backup->recovery_xid <= rt->recovery_target_xid; + + if (rt->time_specified) + return backup->recovery_time <= rt->recovery_target_time; + + if (rt->lsn_specified) + return backup->stop_lsn <= rt->recovery_target_lsn; + + return true; +} + +bool +satisfy_timeline(const parray *timelines, const pgBackup *backup) +{ + int i; + + for (i = 0; i < parray_num(timelines); i++) + { + TimeLineHistoryEntry *timeline; + + timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + if (backup->tli == timeline->tli && + backup->stop_lsn < timeline->end) + return true; + } + return false; +} +/* + * Get recovery options in the string format, parse them + * and fill up the pgRecoveryTarget structure. + */ +pgRecoveryTarget * +parseRecoveryTargetOptions(const char *target_time, + const char *target_xid, + const char *target_inclusive, + TimeLineID target_tli, + const char *target_lsn, + bool target_immediate, + const char *target_name, + const char *target_action, + bool restore_no_validate) +{ + time_t dummy_time; + TransactionId dummy_xid; + bool dummy_bool; + XLogRecPtr dummy_lsn; + /* + * count the number of the mutually exclusive options which may specify + * recovery target. If final value > 1, throw an error. + */ + int recovery_target_specified = 0; + pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); + + /* fill all options with default values */ + rt->time_specified = false; + rt->xid_specified = false; + rt->inclusive_specified = false; + rt->lsn_specified = false; + rt->recovery_target_time = 0; + rt->recovery_target_xid = 0; + rt->recovery_target_lsn = InvalidXLogRecPtr; + rt->target_time_string = NULL; + rt->target_xid_string = NULL; + rt->target_lsn_string = NULL; + rt->recovery_target_inclusive = false; + rt->recovery_target_tli = 0; + rt->recovery_target_immediate = false; + rt->recovery_target_name = NULL; + rt->recovery_target_action = NULL; + rt->restore_no_validate = false; + + /* parse given options */ + if (target_time) + { + recovery_target_specified++; + rt->time_specified = true; + rt->target_time_string = target_time; + + if (parse_time(target_time, &dummy_time, false)) + rt->recovery_target_time = dummy_time; + else + elog(ERROR, "Invalid value of --time option %s", target_time); + } + + if (target_xid) + { + recovery_target_specified++; + rt->xid_specified = true; + rt->target_xid_string = target_xid; + +#ifdef PGPRO_EE + if (parse_uint64(target_xid, &dummy_xid, 0)) +#else + if (parse_uint32(target_xid, &dummy_xid, 0)) +#endif + rt->recovery_target_xid = dummy_xid; + else + elog(ERROR, "Invalid value of --xid option %s", target_xid); + } + + if (target_lsn) + { + recovery_target_specified++; + rt->lsn_specified = true; + rt->target_lsn_string = target_lsn; + if (parse_lsn(target_lsn, &dummy_lsn)) + rt->recovery_target_lsn = dummy_lsn; + else + elog(ERROR, "Invalid value of --lsn option %s", target_lsn); + } + + if (target_inclusive) + { + rt->inclusive_specified = true; + if (parse_bool(target_inclusive, &dummy_bool)) + rt->recovery_target_inclusive = dummy_bool; + else + elog(ERROR, "Invalid value of --inclusive option %s", target_inclusive); + } + + rt->recovery_target_tli = target_tli; + if (target_immediate) + { + recovery_target_specified++; + rt->recovery_target_immediate = target_immediate; + } + + if (restore_no_validate) + { + rt->restore_no_validate = restore_no_validate; + } + + if (target_name) + { + recovery_target_specified++; + rt->recovery_target_name = target_name; + } + + if (target_action) + { + rt->recovery_target_action = target_action; + + if ((strcmp(target_action, "pause") != 0) + && (strcmp(target_action, "promote") != 0) + && (strcmp(target_action, "shutdown") != 0)) + elog(ERROR, "Invalid value of --recovery-target-action option %s", target_action); + } + else + { + /* Default recovery target action is pause */ + rt->recovery_target_action = "pause"; + } + + /* More than one mutually exclusive option was defined. */ + if (recovery_target_specified > 1) + elog(ERROR, "At most one of --immediate, --target-name, --time, --xid, or --lsn can be used"); + + /* If none of the options is defined, '--inclusive' option is meaningless */ + if (!(rt->xid_specified || rt->time_specified || rt->lsn_specified) && rt->recovery_target_inclusive) + elog(ERROR, "--inclusive option applies when either --time or --xid is specified"); + + return rt; +} diff --git a/src/show.c b/src/show.c new file mode 100644 index 00000000..f240ce93 --- /dev/null +++ b/src/show.c @@ -0,0 +1,500 @@ +/*------------------------------------------------------------------------- + * + * show.c: show backup information. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "pqexpbuffer.h" + +#include "utils/json.h" + + +static void show_instance_start(void); +static void show_instance_end(void); +static void show_instance(time_t requested_backup_id, bool show_name); +static int show_backup(time_t requested_backup_id); + +static void show_instance_plain(parray *backup_list, bool show_name); +static void show_instance_json(parray *backup_list); + +static PQExpBufferData show_buf; +static bool first_instance = true; +static int32 json_level = 0; + +int +do_show(time_t requested_backup_id) +{ + if (instance_name == NULL && + requested_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "You must specify --instance to use --backup_id option"); + + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "Cannot open directory \"%s\": %s", + path, strerror(errno)); + + show_instance_start(); + + while (errno = 0, (dent = readdir(dir)) != NULL) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", + child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + + show_instance(INVALID_BACKUP_ID, true); + } + + if (errno) + elog(ERROR, "Cannot read directory \"%s\": %s", + path, strerror(errno)); + + if (closedir(dir)) + elog(ERROR, "Cannot close directory \"%s\": %s", + path, strerror(errno)); + + show_instance_end(); + + return 0; + } + else if (requested_backup_id == INVALID_BACKUP_ID || + show_format == SHOW_JSON) + { + show_instance_start(); + show_instance(requested_backup_id, false); + show_instance_end(); + + return 0; + } + else + return show_backup(requested_backup_id); +} + +static void +pretty_size(int64 size, char *buf, size_t len) +{ + int exp = 0; + + /* minus means the size is invalid */ + if (size < 0) + { + strncpy(buf, "----", len); + return; + } + + /* determine postfix */ + while (size > 9999) + { + ++exp; + size /= 1000; + } + + switch (exp) + { + case 0: + snprintf(buf, len, "%dB", (int) size); + break; + case 1: + snprintf(buf, len, "%dkB", (int) size); + break; + case 2: + snprintf(buf, len, "%dMB", (int) size); + break; + case 3: + snprintf(buf, len, "%dGB", (int) size); + break; + case 4: + snprintf(buf, len, "%dTB", (int) size); + break; + case 5: + snprintf(buf, len, "%dPB", (int) size); + break; + default: + strncpy(buf, "***", len); + break; + } +} + +static TimeLineID +get_parent_tli(TimeLineID child_tli) +{ + TimeLineID result = 0; + char path[MAXPGPATH]; + char fline[MAXPGPATH]; + FILE *fd; + + /* Timeline 1 does not have a history file and parent timeline */ + if (child_tli == 1) + return 0; + + /* Search history file in archives */ + snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, + child_tli); + fd = fopen(path, "rt"); + if (fd == NULL) + { + if (errno != ENOENT) + elog(ERROR, "could not open file \"%s\": %s", path, + strerror(errno)); + + /* Did not find history file, do not raise the error */ + return 0; + } + + /* + * Parse the file... + */ + while (fgets(fline, sizeof(fline), fd) != NULL) + { + /* skip leading whitespace and check for # comment */ + char *ptr; + char *endptr; + + for (ptr = fline; *ptr; ptr++) + { + if (!IsSpace(*ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + /* expect a numeric timeline ID as first field of line */ + result = (TimeLineID) strtoul(ptr, &endptr, 0); + if (endptr == ptr) + elog(ERROR, + "syntax error(timeline ID) in history file: %s", + fline); + } + + fclose(fd); + + /* TLI of the last line is parent TLI */ + return result; +} + +/* + * Initialize instance visualization. + */ +static void +show_instance_start(void) +{ + initPQExpBuffer(&show_buf); + + if (show_format == SHOW_PLAIN) + return; + + first_instance = true; + json_level = 0; + + appendPQExpBufferChar(&show_buf, '['); + json_level++; +} + +/* + * Finalize instance visualization. + */ +static void +show_instance_end(void) +{ + if (show_format == SHOW_JSON) + appendPQExpBufferStr(&show_buf, "\n]\n"); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show brief meta information about all backups in the backup instance. + */ +static void +show_instance(time_t requested_backup_id, bool show_name) +{ + parray *backup_list; + + backup_list = catalog_get_backup_list(requested_backup_id); + + if (show_format == SHOW_PLAIN) + show_instance_plain(backup_list, show_name); + else if (show_format == SHOW_JSON) + show_instance_json(backup_list); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* + * Show detailed meta information about specified backup. + */ +static int +show_backup(time_t requested_backup_id) +{ + pgBackup *backup; + + backup = read_backup(requested_backup_id); + if (backup == NULL) + { + elog(INFO, "Requested backup \"%s\" is not found.", + /* We do not need free base36enc's result, we exit anyway */ + base36enc(requested_backup_id)); + /* This is not error */ + return 0; + } + + if (show_format == SHOW_PLAIN) + pgBackupWriteControl(stdout, backup); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + pgBackupFree(backup); + + return 0; +} + +/* + * Plain output. + */ + +/* + * Show instance backups in plain format. + */ +static void +show_instance_plain(parray *backup_list, bool show_name) +{ + int i; + + if (show_name) + printfPQExpBuffer(&show_buf, "\nBACKUP INSTANCE '%s'\n", instance_name); + + /* if you add new fields here, fix the header */ + /* show header */ + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + appendPQExpBufferStr(&show_buf, + " Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n"); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char duration[20] = "----"; + char data_bytes_str[10] = "----"; + + if (backup->recovery_time != (time_t) 0) + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + if (backup->end_time != (time_t) 0) + snprintf(duration, lengthof(duration), "%.*lfs", 0, + difftime(backup->end_time, backup->start_time)); + + /* + * Calculate Data field, in the case of full backup this shows the + * total amount of data. For an differential backup, this size is only + * the difference of data accumulated. + */ + pretty_size(backup->data_bytes, data_bytes_str, + lengthof(data_bytes_str)); + + /* Get parent timeline before printing */ + parent_tli = get_parent_tli(backup->tli); + + appendPQExpBuffer(&show_buf, + " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, + (backup->server_version[0] ? backup->server_version : "----"), + base36enc(backup->start_time), + timestamp, + pgBackupGetBackupMode(backup), + backup->stream ? "STREAM": "ARCHIVE", + backup->tli, + parent_tli, + duration, + data_bytes_str, + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn, + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn, + status2str(backup->status)); + } +} + +/* + * Json output. + */ + +/* + * Show instance backups in json format. + */ +static void +show_instance_json(parray *backup_list) +{ + int i; + PQExpBuffer buf = &show_buf; + + if (!first_instance) + appendPQExpBufferChar(buf, ','); + + /* Begin of instance object */ + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "instance", instance_name, json_level, false); + json_add_key(buf, "backups", json_level, true); + + /* + * List backups. + */ + json_add(buf, JT_BEGIN_ARRAY, &json_level); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char lsn[20]; + + if (i != 0) + appendPQExpBufferChar(buf, ','); + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "id", base36enc(backup->start_time), json_level, + false); + + if (backup->parent_backup != 0) + json_add_value(buf, "parent-backup-id", + base36enc(backup->parent_backup), json_level, true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_level, true); + + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", + json_level, true); + + json_add_value(buf, "compress-alg", + deparse_compress_alg(backup->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", backup->compress_level); + + json_add_value(buf, "from-replica", + backup->from_replica ? "true" : "false", json_level, + true); + + json_add_key(buf, "block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->block_size); + + json_add_key(buf, "xlog-block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->wal_block_size); + + json_add_key(buf, "checksum-version", json_level, true); + appendPQExpBuffer(buf, "%u", backup->checksum_version); + + json_add_value(buf, "program-version", backup->program_version, + json_level, true); + json_add_value(buf, "server-version", backup->server_version, + json_level, true); + + json_add_key(buf, "current-tli", json_level, true); + appendPQExpBuffer(buf, "%d", backup->tli); + + json_add_key(buf, "parent-tli", json_level, true); + parent_tli = get_parent_tli(backup->tli); + appendPQExpBuffer(buf, "%u", parent_tli); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); + json_add_value(buf, "start-lsn", lsn, json_level, true); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); + json_add_value(buf, "stop-lsn", lsn, json_level, true); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + json_add_value(buf, "start-time", timestamp, json_level, true); + + if (backup->end_time) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, json_level, true); + } + + json_add_key(buf, "recovery-xid", json_level, true); + appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, json_level, true); + } + + if (backup->data_bytes != BYTES_INVALID) + { + json_add_key(buf, "data-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + } + + if (backup->wal_bytes != BYTES_INVALID) + { + json_add_key(buf, "wal-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); + } + + if (backup->primary_conninfo) + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, + json_level, true); + + json_add_value(buf, "status", status2str(backup->status), json_level, + true); + + json_add(buf, JT_END_OBJECT, &json_level); + } + + /* End of backups */ + json_add(buf, JT_END_ARRAY, &json_level); + + /* End of instance object */ + json_add(buf, JT_END_OBJECT, &json_level); + + first_instance = false; +} diff --git a/src/status.c b/src/status.c new file mode 100644 index 00000000..155a07f4 --- /dev/null +++ b/src/status.c @@ -0,0 +1,118 @@ +/*------------------------------------------------------------------------- + * + * status.c + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * + * Monitor status of a PostgreSQL server. + * + *------------------------------------------------------------------------- + */ + + +#include "postgres_fe.h" + +#include +#include +#include + +#include "pg_probackup.h" + +/* PID can be negative for standalone backend */ +typedef long pgpid_t; + +static pgpid_t get_pgpid(void); +static bool postmaster_is_alive(pid_t pid); + +/* + * get_pgpid + * + * Get PID of postmaster, by scanning postmaster.pid. + */ +static pgpid_t +get_pgpid(void) +{ + FILE *pidf; + long pid; + char pid_file[MAXPGPATH]; + + snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); + + pidf = fopen(pid_file, PG_BINARY_R); + if (pidf == NULL) + { + /* No pid file, not an error on startup */ + if (errno == ENOENT) + return 0; + else + { + elog(ERROR, "could not open PID file \"%s\": %s", + pid_file, strerror(errno)); + } + } + if (fscanf(pidf, "%ld", &pid) != 1) + { + /* Is the file empty? */ + if (ftell(pidf) == 0 && feof(pidf)) + elog(ERROR, "the PID file \"%s\" is empty", + pid_file); + else + elog(ERROR, "invalid data in PID file \"%s\"\n", + pid_file); + } + fclose(pidf); + return (pgpid_t) pid; +} + +/* + * postmaster_is_alive + * + * Check whether postmaster is alive or not. + */ +static bool +postmaster_is_alive(pid_t pid) +{ + /* + * Test to see if the process is still there. Note that we do not + * consider an EPERM failure to mean that the process is still there; + * EPERM must mean that the given PID belongs to some other userid, and + * considering the permissions on $PGDATA, that means it's not the + * postmaster we are after. + * + * Don't believe that our own PID or parent shell's PID is the postmaster, + * either. (Windows hasn't got getppid(), though.) + */ + if (pid == getpid()) + return false; +#ifndef WIN32 + if (pid == getppid()) + return false; +#endif + if (kill(pid, 0) == 0) + return true; + return false; +} + +/* + * is_pg_running + * + * + */ +bool +is_pg_running(void) +{ + pgpid_t pid; + + pid = get_pgpid(); + + /* 0 means no pid file */ + if (pid == 0) + return false; + + /* Case of a standalone backend */ + if (pid < 0) + pid = -pid; + + /* Check if postmaster is alive */ + return postmaster_is_alive((pid_t) pid); +} diff --git a/src/util.c b/src/util.c new file mode 100644 index 00000000..82814d11 --- /dev/null +++ b/src/util.c @@ -0,0 +1,349 @@ +/*------------------------------------------------------------------------- + * + * util.c: log messages to log file or stderr, and misc code. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include + +#include "storage/bufpage.h" +#if PG_VERSION_NUM >= 110000 +#include "streamutil.h" +#endif + +const char * +base36enc(long unsigned int value) +{ + const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ + static char buffer[14]; + unsigned int offset = sizeof(buffer); + + buffer[--offset] = '\0'; + do { + buffer[--offset] = base36[value % 36]; + } while (value /= 36); + + return &buffer[offset]; +} + +/* + * Same as base36enc(), but the result must be released by the user. + */ +char * +base36enc_dup(long unsigned int value) +{ + const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ + char buffer[14]; + unsigned int offset = sizeof(buffer); + + buffer[--offset] = '\0'; + do { + buffer[--offset] = base36[value % 36]; + } while (value /= 36); + + return strdup(&buffer[offset]); +} + +long unsigned int +base36dec(const char *text) +{ + return strtoul(text, NULL, 36); +} + +static void +checkControlFile(ControlFileData *ControlFile) +{ + pg_crc32c crc; + + /* Calculate CRC */ + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *) ControlFile, offsetof(ControlFileData, crc)); + FIN_CRC32C(crc); + + /* Then compare it */ + if (!EQ_CRC32C(crc, ControlFile->crc)) + elog(ERROR, "Calculated CRC checksum does not match value stored in file.\n" + "Either the file is corrupt, or it has a different layout than this program\n" + "is expecting. The results below are untrustworthy."); + + if (ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0) + elog(ERROR, "possible byte ordering mismatch\n" + "The byte ordering used to store the pg_control file might not match the one\n" + "used by this program. In that case the results below would be incorrect, and\n" + "the PostgreSQL installation would be incompatible with this data directory."); +} + +/* + * Verify control file contents in the buffer src, and copy it to *ControlFile. + */ +static void +digestControlFile(ControlFileData *ControlFile, char *src, size_t size) +{ +#if PG_VERSION_NUM >= 100000 + int ControlFileSize = PG_CONTROL_FILE_SIZE; +#else + int ControlFileSize = PG_CONTROL_SIZE; +#endif + + if (size != ControlFileSize) + elog(ERROR, "unexpected control file size %d, expected %d", + (int) size, ControlFileSize); + + memcpy(ControlFile, src, sizeof(ControlFileData)); + + /* Additional checks on control file */ + checkControlFile(ControlFile); +} + +/* + * Utility shared by backup and restore to fetch the current timeline + * used by a node. + */ +TimeLineID +get_current_timeline(bool safe) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + if (safe && buffer == NULL) + return 0; + + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.checkPointCopy.ThisTimeLineID; +} + +uint64 +get_system_identifier(char *pgdata_path) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.system_identifier; +} + +uint64 +get_remote_system_identifier(PGconn *conn) +{ +#if PG_VERSION_NUM >= 90600 + PGresult *res; + uint64 system_id_conn; + char *val; + + res = pgut_execute(conn, + "SELECT system_identifier FROM pg_catalog.pg_control_system()", + 0, NULL); + val = PQgetvalue(res, 0, 0); + if (!parse_uint64(val, &system_id_conn, 0)) + { + PQclear(res); + elog(ERROR, "%s is not system_identifier", val); + } + PQclear(res); + + return system_id_conn; +#else + char *buffer; + size_t size; + ControlFileData ControlFile; + + buffer = fetchFile(conn, "global/pg_control", &size); + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.system_identifier; +#endif +} + +uint32 +get_xlog_seg_size(char *pgdata_path) +{ +#if PG_VERSION_NUM >= 110000 + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.xlog_seg_size; +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + +uint32 +get_data_checksum_version(bool safe) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.data_checksum_version; +} + + +/* + * Convert time_t value to ISO-8601 format string. Always set timezone offset. + */ +void +time2iso(char *buf, size_t len, time_t time) +{ + struct tm *ptm = gmtime(&time); + time_t gmt = mktime(ptm); + time_t offset; + char *ptr = buf; + + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); + + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", + (offset >= 0) ? '+' : '-', + abs((int) offset) / SECS_PER_HOUR); + + if (abs((int) offset) % SECS_PER_HOUR != 0) + { + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", + abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); + } +} + +/* copied from timestamp.c */ +pg_time_t +timestamptz_to_time_t(TimestampTz t) +{ + pg_time_t result; + +#ifdef HAVE_INT64_TIMESTAMP + result = (pg_time_t) (t / USECS_PER_SEC + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); +#else + result = (pg_time_t) (t + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); +#endif + return result; +} + +/* Parse string representation of the server version */ +int +parse_server_version(char *server_version_str) +{ + int nfields; + int result = 0; + int major_version = 0; + int minor_version = 0; + + nfields = sscanf(server_version_str, "%d.%d", &major_version, &minor_version); + if (nfields == 2) + { + /* Server version lower than 10 */ + if (major_version > 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000 + minor_version * 100; + } + else if (nfields == 1) + { + if (major_version < 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000; + } + else + elog(ERROR, "Unknown server version format"); + + return result; +} + +const char * +status2str(BackupStatus status) +{ + static const char *statusName[] = + { + "UNKNOWN", + "OK", + "ERROR", + "RUNNING", + "MERGING", + "DELETING", + "DELETED", + "DONE", + "ORPHAN", + "CORRUPT" + }; + if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status) + return "UNKNOWN"; + + return statusName[status]; +} + +void +remove_trailing_space(char *buf, int comment_mark) +{ + int i; + char *last_char = NULL; + + for (i = 0; buf[i]; i++) + { + if (buf[i] == comment_mark || buf[i] == '\n' || buf[i] == '\r') + { + buf[i] = '\0'; + break; + } + } + for (i = 0; buf[i]; i++) + { + if (!isspace(buf[i])) + last_char = buf + i; + } + if (last_char != NULL) + *(last_char + 1) = '\0'; + +} + +void +remove_not_digit(char *buf, size_t len, const char *str) +{ + int i, j; + + for (i = 0, j = 0; str[i] && j < len; i++) + { + if (!isdigit(str[i])) + continue; + buf[j++] = str[i]; + } + buf[j] = '\0'; +} diff --git a/src/utils/json.c b/src/utils/json.c new file mode 100644 index 00000000..3afbe9e7 --- /dev/null +++ b/src/utils/json.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * json.c: - make json document. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "json.h" + +static void json_add_indent(PQExpBuffer buf, int32 level); +static void json_add_escaped(PQExpBuffer buf, const char *str); + +/* + * Start or end json token. Currently it is a json object or array. + * + * Function modifies level value and adds indent if it appropriate. + */ +void +json_add(PQExpBuffer buf, JsonToken type, int32 *level) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + *level += 1; + break; + case JT_END_ARRAY: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '{'); + *level += 1; + break; + case JT_END_OBJECT: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; + } +} + +/* + * Add json object's key. If it isn't first key we need to add a comma. + */ +void +json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf, level); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +/* + * Add json object's key and value. If it isn't first key we need to add a + * comma. + */ +void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma) +{ + json_add_key(buf, name, level, add_comma); + json_add_escaped(buf, value); +} + +static void +json_add_indent(PQExpBuffer buf, int32 level) +{ + uint16 i; + + if (level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < level; i++) + appendPQExpBufferStr(buf, " "); +} + +static void +json_add_escaped(PQExpBuffer buf, const char *str) +{ + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} diff --git a/src/utils/json.h b/src/utils/json.h new file mode 100644 index 00000000..cf5a7064 --- /dev/null +++ b/src/utils/json.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * json.h: - prototypes of json output functions. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_JSON_H +#define PROBACKUP_JSON_H + +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +/* + * Json document tokens. + */ +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_key(PQExpBuffer buf, const char *name, int32 level, + bool add_comma); +extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma); + +#endif /* PROBACKUP_JSON_H */ diff --git a/src/utils/logger.c b/src/utils/logger.c new file mode 100644 index 00000000..31669ed0 --- /dev/null +++ b/src/utils/logger.c @@ -0,0 +1,621 @@ +/*------------------------------------------------------------------------- + * + * logger.c: - log events into log file or stderr. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +#include "logger.h" +#include "pgut.h" +#include "pg_probackup.h" +#include "thread.h" + +/* Logger parameters */ + +int log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; +int log_level_file = LOG_LEVEL_FILE_DEFAULT; + +char *log_filename = NULL; +char *error_log_filename = NULL; +char *log_directory = NULL; +/* + * If log_path is empty logging is not initialized. + * We will log only into stderr + */ +char log_path[MAXPGPATH] = ""; + +/* Maximum size of an individual log file in kilobytes */ +int log_rotation_size = 0; +/* Maximum lifetime of an individual log file in minutes */ +int log_rotation_age = 0; + +/* Implementation for logging.h */ + +typedef enum +{ + PG_DEBUG, + PG_PROGRESS, + PG_WARNING, + PG_FATAL +} eLogType; + +void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); + +static void elog_internal(int elevel, bool file_only, const char *fmt, va_list args) + pg_attribute_printf(3, 0); +static void elog_stderr(int elevel, const char *fmt, ...) + pg_attribute_printf(2, 3); + +/* Functions to work with log files */ +static void open_logfile(FILE **file, const char *filename_format); +static void release_logfile(void); +static char *logfile_getname(const char *format, time_t timestamp); +static FILE *logfile_open(const char *filename, const char *mode); + +/* Static variables */ + +static FILE *log_file = NULL; +static FILE *error_log_file = NULL; + +static bool exit_hook_registered = false; +/* Logging of the current thread is in progress */ +static bool loggin_in_progress = false; + +static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; + +void +init_logger(const char *root_path) +{ + /* Set log path */ + if (log_level_file != LOG_OFF || error_log_filename) + { + if (log_directory) + strcpy(log_path, log_directory); + else + join_path_components(log_path, root_path, LOG_DIRECTORY_DEFAULT); + } +} + +static void +write_elevel(FILE *stream, int elevel) +{ + switch (elevel) + { + case VERBOSE: + fputs("VERBOSE: ", stream); + break; + case LOG: + fputs("LOG: ", stream); + break; + case INFO: + fputs("INFO: ", stream); + break; + case NOTICE: + fputs("NOTICE: ", stream); + break; + case WARNING: + fputs("WARNING: ", stream); + break; + case ERROR: + fputs("ERROR: ", stream); + break; + default: + elog_stderr(ERROR, "invalid logging level: %d", elevel); + break; + } +} + +/* + * Exit with code if it is an error. + * Check for in_cleanup flag to avoid deadlock in case of ERROR in cleanup + * routines. + */ +static void +exit_if_necessary(int elevel) +{ + if (elevel > WARNING && !in_cleanup) + { + /* Interrupt other possible routines */ + interrupted = true; + + if (loggin_in_progress) + { + loggin_in_progress = false; + pthread_mutex_unlock(&log_file_mutex); + } + + /* If this is not the main thread then don't call exit() */ + if (main_tid != pthread_self()) +#ifdef WIN32 + ExitThread(elevel); +#else + pthread_exit(NULL); +#endif + else + exit(elevel); + } +} + +/* + * Logs to stderr or to log file and exit if ERROR. + * + * Actual implementation for elog() and pg_log(). + */ +static void +elog_internal(int elevel, bool file_only, const char *fmt, va_list args) +{ + bool write_to_file, + write_to_error_log, + write_to_stderr; + va_list error_args, + std_args; + time_t log_time = (time_t) time(NULL); + char strfbuf[128]; + + write_to_file = elevel >= log_level_file && log_path[0] != '\0'; + write_to_error_log = elevel >= ERROR && error_log_filename && + log_path[0] != '\0'; + write_to_stderr = elevel >= log_level_console && !file_only; + + pthread_lock(&log_file_mutex); +#ifdef WIN32 + std_args = NULL; + error_args = NULL; +#endif + loggin_in_progress = true; + + /* We need copy args only if we need write to error log file */ + if (write_to_error_log) + va_copy(error_args, args); + /* + * We need copy args only if we need write to stderr. But do not copy args + * if we need to log only to stderr. + */ + if (write_to_stderr && write_to_file) + va_copy(std_args, args); + + if (write_to_file || write_to_error_log) + strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", + localtime(&log_time)); + + /* + * Write message to log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_file) + { + if (log_file == NULL) + { + if (log_filename == NULL) + open_logfile(&log_file, LOG_FILENAME_DEFAULT); + else + open_logfile(&log_file, log_filename); + } + + fprintf(log_file, "%s: ", strfbuf); + write_elevel(log_file, elevel); + + vfprintf(log_file, fmt, args); + fputc('\n', log_file); + fflush(log_file); + } + + /* + * Write error message to error log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_error_log) + { + if (error_log_file == NULL) + open_logfile(&error_log_file, error_log_filename); + + fprintf(error_log_file, "%s: ", strfbuf); + write_elevel(error_log_file, elevel); + + vfprintf(error_log_file, fmt, error_args); + fputc('\n', error_log_file); + fflush(error_log_file); + + va_end(error_args); + } + + /* + * Write to stderr if the message was not written to log file. + * Write to stderr if the message level is greater than WARNING anyway. + */ + if (write_to_stderr) + { + write_elevel(stderr, elevel); + if (write_to_file) + vfprintf(stderr, fmt, std_args); + else + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + if (write_to_file) + va_end(std_args); + } + + exit_if_necessary(elevel); + + loggin_in_progress = false; + pthread_mutex_unlock(&log_file_mutex); +} + +/* + * Log only to stderr. It is called only within elog_internal() when another + * logging already was started. + */ +static void +elog_stderr(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < ERROR) + return; + + va_start(args, fmt); + + write_elevel(stderr, elevel); + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + va_end(args); + + exit_if_necessary(elevel); +} + +/* + * Logs to stderr or to log file and exit if ERROR. + */ +void +elog(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, false, fmt, args); + va_end(args); +} + +/* + * Logs only to log file and exit if ERROR. + */ +void +elog_file(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, true, fmt, args); + va_end(args); +} + +/* + * Implementation of pg_log() from logging.h. + */ +void +pg_log(eLogType type, const char *fmt, ...) +{ + va_list args; + int elevel = INFO; + + /* Transform logging level from eLogType to utils/logger.h levels */ + switch (type) + { + case PG_DEBUG: + elevel = LOG; + break; + case PG_PROGRESS: + elevel = INFO; + break; + case PG_WARNING: + elevel = WARNING; + break; + case PG_FATAL: + elevel = ERROR; + break; + default: + elog(ERROR, "invalid logging level: %d", type); + break; + } + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, false, fmt, args); + va_end(args); +} + +/* + * Parses string representation of log level. + */ +int +parse_log_level(const char *level) +{ + const char *v = level; + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*v)) + v++; + len = strlen(v); + + if (len == 0) + elog(ERROR, "log-level is empty"); + + if (pg_strncasecmp("off", v, len) == 0) + return LOG_OFF; + else if (pg_strncasecmp("verbose", v, len) == 0) + return VERBOSE; + else if (pg_strncasecmp("log", v, len) == 0) + return LOG; + else if (pg_strncasecmp("info", v, len) == 0) + return INFO; + else if (pg_strncasecmp("notice", v, len) == 0) + return NOTICE; + else if (pg_strncasecmp("warning", v, len) == 0) + return WARNING; + else if (pg_strncasecmp("error", v, len) == 0) + return ERROR; + + /* Log level is invalid */ + elog(ERROR, "invalid log-level \"%s\"", level); + return 0; +} + +/* + * Converts integer representation of log level to string. + */ +const char * +deparse_log_level(int level) +{ + switch (level) + { + case LOG_OFF: + return "OFF"; + case VERBOSE: + return "VERBOSE"; + case LOG: + return "LOG"; + case INFO: + return "INFO"; + case NOTICE: + return "NOTICE"; + case WARNING: + return "WARNING"; + case ERROR: + return "ERROR"; + default: + elog(ERROR, "invalid log-level %d", level); + } + + return NULL; +} + +/* + * Construct logfile name using timestamp information. + * + * Result is palloc'd. + */ +static char * +logfile_getname(const char *format, time_t timestamp) +{ + char *filename; + size_t len; + struct tm *tm = localtime(×tamp); + + if (log_path[0] == '\0') + elog_stderr(ERROR, "logging path is not set"); + + filename = (char *) palloc(MAXPGPATH); + + snprintf(filename, MAXPGPATH, "%s/", log_path); + + len = strlen(filename); + + /* Treat log_filename as a strftime pattern */ + if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) + elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); + + return filename; +} + +/* + * Open a new log file. + */ +static FILE * +logfile_open(const char *filename, const char *mode) +{ + FILE *fh; + + /* + * Create log directory if not present; ignore errors + */ + mkdir(log_path, S_IRWXU); + + fh = fopen(filename, mode); + + if (fh) + setvbuf(fh, NULL, PG_IOLBF, 0); + else + { + int save_errno = errno; + + elog_stderr(ERROR, "could not open log file \"%s\": %s", + filename, strerror(errno)); + errno = save_errno; + } + + return fh; +} + +/* + * Open the log file. + */ +static void +open_logfile(FILE **file, const char *filename_format) +{ + char *filename; + char control[MAXPGPATH]; + struct stat st; + FILE *control_file; + time_t cur_time = time(NULL); + bool rotation_requested = false, + logfile_exists = false; + + filename = logfile_getname(filename_format, cur_time); + + /* "log_path" was checked in logfile_getname() */ + snprintf(control, MAXPGPATH, "%s.rotation", filename); + + if (stat(filename, &st) == -1) + { + if (errno == ENOENT) + { + /* There is no file "filename" and rotation does not need */ + goto logfile_open; + } + else + elog_stderr(ERROR, "cannot stat log file \"%s\": %s", + filename, strerror(errno)); + } + /* Found log file "filename" */ + logfile_exists = true; + + /* First check for rotation */ + if (log_rotation_size > 0 || log_rotation_age > 0) + { + /* Check for rotation by age */ + if (log_rotation_age > 0) + { + struct stat control_st; + + if (stat(control, &control_st) == -1) + { + if (errno != ENOENT) + elog_stderr(ERROR, "cannot stat rotation file \"%s\": %s", + control, strerror(errno)); + } + else + { + char buf[1024]; + + control_file = fopen(control, "r"); + if (control_file == NULL) + elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + if (fgets(buf, lengthof(buf), control_file)) + { + time_t creation_time; + + if (!parse_int64(buf, (int64 *) &creation_time, 0)) + elog_stderr(ERROR, "rotation file \"%s\" has wrong " + "creation timestamp \"%s\"", + control, buf); + /* Parsed creation time */ + + rotation_requested = (cur_time - creation_time) > + /* convert to seconds */ + log_rotation_age * 60; + } + else + elog_stderr(ERROR, "cannot read creation timestamp from " + "rotation file \"%s\"", control); + + fclose(control_file); + } + } + + /* Check for rotation by size */ + if (!rotation_requested && log_rotation_size > 0) + rotation_requested = st.st_size >= + /* convert to bytes */ + log_rotation_size * 1024L; + } + +logfile_open: + if (rotation_requested) + *file = logfile_open(filename, "w"); + else + *file = logfile_open(filename, "a"); + pfree(filename); + + /* Rewrite rotation control file */ + if (rotation_requested || !logfile_exists) + { + time_t timestamp = time(NULL); + + control_file = fopen(control, "w"); + if (control_file == NULL) + elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + fprintf(control_file, "%ld", timestamp); + + fclose(control_file); + } + + /* + * Arrange to close opened file at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(release_logfile); + exit_hook_registered = true; + } +} + +/* + * Closes opened file. + */ +static void +release_logfile(void) +{ + if (log_file) + { + fclose(log_file); + log_file = NULL; + } + if (error_log_file) + { + fclose(error_log_file); + error_log_file = NULL; + } +} diff --git a/src/utils/logger.h b/src/utils/logger.h new file mode 100644 index 00000000..8643ad18 --- /dev/null +++ b/src/utils/logger.h @@ -0,0 +1,54 @@ +/*------------------------------------------------------------------------- + * + * logger.h: - prototypes of logger functions. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef LOGGER_H +#define LOGGER_H + +#include "postgres_fe.h" + +#define LOG_NONE (-10) + +/* Log level */ +#define VERBOSE (-5) +#define LOG (-4) +#define INFO (-3) +#define NOTICE (-2) +#define WARNING (-1) +#define ERROR 1 +#define LOG_OFF 10 + +/* Logger parameters */ + +extern int log_to_file; +extern int log_level_console; +extern int log_level_file; + +extern char *log_filename; +extern char *error_log_filename; +extern char *log_directory; +extern char log_path[MAXPGPATH]; + +#define LOG_ROTATION_SIZE_DEFAULT 0 +#define LOG_ROTATION_AGE_DEFAULT 0 +extern int log_rotation_size; +extern int log_rotation_age; + +#define LOG_LEVEL_CONSOLE_DEFAULT INFO +#define LOG_LEVEL_FILE_DEFAULT LOG_OFF + +#undef elog +extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); +extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); + +extern void init_logger(const char *root_path); + +extern int parse_log_level(const char *level); +extern const char *deparse_log_level(int level); + +#endif /* LOGGER_H */ diff --git a/src/utils/parray.c b/src/utils/parray.c new file mode 100644 index 00000000..a9ba7c8e --- /dev/null +++ b/src/utils/parray.c @@ -0,0 +1,196 @@ +/*------------------------------------------------------------------------- + * + * parray.c: pointer array collection. + * + * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "src/pg_probackup.h" + +/* members of struct parray are hidden from client. */ +struct parray +{ + void **data; /* poiter array, expanded if necessary */ + size_t alloced; /* number of elements allocated */ + size_t used; /* number of elements in use */ +}; + +/* + * Create new parray object. + * Never returns NULL. + */ +parray * +parray_new(void) +{ + parray *a = pgut_new(parray); + + a->data = NULL; + a->used = 0; + a->alloced = 0; + + parray_expand(a, 1024); + + return a; +} + +/* + * Expand array pointed by data to newsize. + * Elements in expanded area are initialized to NULL. + * Note: never returns NULL. + */ +void +parray_expand(parray *array, size_t newsize) +{ + void **p; + + /* already allocated */ + if (newsize <= array->alloced) + return; + + p = pgut_realloc(array->data, sizeof(void *) * newsize); + + /* initialize expanded area to NULL */ + memset(p + array->alloced, 0, (newsize - array->alloced) * sizeof(void *)); + + array->alloced = newsize; + array->data = p; +} + +void +parray_free(parray *array) +{ + if (array == NULL) + return; + free(array->data); + free(array); +} + +void +parray_append(parray *array, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + array->data[array->used++] = elem; +} + +void +parray_insert(parray *array, size_t index, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + memmove(array->data + index + 1, array->data + index, + (array->alloced - index - 1) * sizeof(void *)); + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; + else + array->used++; +} + +/* + * Concatinate two parray. + * parray_concat() appends the copy of the content of src to the end of dest. + */ +parray * +parray_concat(parray *dest, const parray *src) +{ + /* expand head array */ + parray_expand(dest, dest->used + src->used); + + /* copy content of src after content of dest */ + memcpy(dest->data + dest->used, src->data, src->used * sizeof(void *)); + dest->used += parray_num(src); + + return dest; +} + +void +parray_set(parray *array, size_t index, void *elem) +{ + if (index > array->alloced - 1) + parray_expand(array, index + 1); + + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; +} + +void * +parray_get(const parray *array, size_t index) +{ + if (index > array->alloced - 1) + return NULL; + return array->data[index]; +} + +void * +parray_remove(parray *array, size_t index) +{ + void *val; + + /* removing unused element */ + if (index > array->used) + return NULL; + + val = array->data[index]; + + /* Do not move if the last element was removed. */ + if (index < array->alloced - 1) + memmove(array->data + index, array->data + index + 1, + (array->alloced - index - 1) * sizeof(void *)); + + /* adjust used count */ + array->used--; + + return val; +} + +bool +parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + int i; + + for (i = 0; i < array->used; i++) + { + if (compare(&key, &array->data[i]) == 0) + { + parray_remove(array, i); + return true; + } + } + return false; +} + +size_t +parray_num(const parray *array) +{ + return array->used; +} + +void +parray_qsort(parray *array, int(*compare)(const void *, const void *)) +{ + qsort(array->data, array->used, sizeof(void *), compare); +} + +void +parray_walk(parray *array, void (*action)(void *)) +{ + int i; + for (i = 0; i < array->used; i++) + action(array->data[i]); +} + +void * +parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + return bsearch(&key, array->data, array->used, sizeof(void *), compare); +} diff --git a/src/utils/parray.h b/src/utils/parray.h new file mode 100644 index 00000000..833a6961 --- /dev/null +++ b/src/utils/parray.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * parray.h: pointer array collection. + * + * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#ifndef PARRAY_H +#define PARRAY_H + +/* + * "parray" hold pointers to objects in a linear memory area. + * Client use "parray *" to access parray object. + */ +typedef struct parray parray; + +extern parray *parray_new(void); +extern void parray_expand(parray *array, size_t newnum); +extern void parray_free(parray *array); +extern void parray_append(parray *array, void *val); +extern void parray_insert(parray *array, size_t index, void *val); +extern parray *parray_concat(parray *head, const parray *tail); +extern void parray_set(parray *array, size_t index, void *val); +extern void *parray_get(const parray *array, size_t index); +extern void *parray_remove(parray *array, size_t index); +extern bool parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern size_t parray_num(const parray *array); +extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); +extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern void parray_walk(parray *array, void (*action)(void *)); + +#endif /* PARRAY_H */ + diff --git a/src/utils/pgut.c b/src/utils/pgut.c new file mode 100644 index 00000000..f341c6a4 --- /dev/null +++ b/src/utils/pgut.c @@ -0,0 +1,2417 @@ +/*------------------------------------------------------------------------- + * + * pgut.c + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" +#include "libpq/pqsignal.h" + +#include "getopt_long.h" +#include +#include +#include + +#include "logger.h" +#include "pgut.h" + +/* old gcc doesn't have LLONG_MAX. */ +#ifndef LLONG_MAX +#if defined(HAVE_LONG_INT_64) || !defined(HAVE_LONG_LONG_INT_64) +#define LLONG_MAX LONG_MAX +#else +#define LLONG_MAX INT64CONST(0x7FFFFFFFFFFFFFFF) +#endif +#endif + +#define MAX_TZDISP_HOUR 15 /* maximum allowed hour part */ +#define SECS_PER_MINUTE 60 +#define MINS_PER_HOUR 60 +#define MAXPG_LSNCOMPONENT 8 + +const char *PROGRAM_NAME = NULL; + +const char *pgut_dbname = NULL; +const char *host = NULL; +const char *port = NULL; +const char *username = NULL; +static char *password = NULL; +bool prompt_password = true; +bool force_password = false; + +/* Database connections */ +static PGcancel *volatile cancel_conn = NULL; + +/* Interrupted by SIGINT (Ctrl+C) ? */ +bool interrupted = false; +bool in_cleanup = false; +bool in_password = false; + +static bool parse_pair(const char buffer[], char key[], char value[]); + +/* Connection routines */ +static void init_cancel_handler(void); +static void on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn); +static void on_after_exec(PGcancel *thread_cancel_conn); +static void on_interrupt(void); +static void on_cleanup(void); +static void exit_or_abort(int exitcode); +static const char *get_username(void); +static pqsigfunc oldhandler = NULL; + +/* + * Unit conversion tables. + * + * Copied from guc.c. + */ +#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ + +typedef struct +{ + char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or + * "min" */ + int base_unit; /* OPTION_UNIT_XXX */ + int multiplier; /* If positive, multiply the value with this + * for unit -> base_unit conversion. If + * negative, divide (with the absolute value) */ +} unit_conversion; + +static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; + +static const unit_conversion memory_unit_conversion_table[] = +{ + {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, + {"GB", OPTION_UNIT_KB, 1024 * 1024}, + {"MB", OPTION_UNIT_KB, 1024}, + {"KB", OPTION_UNIT_KB, 1}, + {"kB", OPTION_UNIT_KB, 1}, + + {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, + + {""} /* end of table marker */ +}; + +static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; + +static const unit_conversion time_unit_conversion_table[] = +{ + {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, + {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, + {"min", OPTION_UNIT_MS, 1000 * 60}, + {"s", OPTION_UNIT_MS, 1000}, + {"ms", OPTION_UNIT_MS, 1}, + + {"d", OPTION_UNIT_S, 60 * 60 * 24}, + {"h", OPTION_UNIT_S, 60 * 60}, + {"min", OPTION_UNIT_S, 60}, + {"s", OPTION_UNIT_S, 1}, + {"ms", OPTION_UNIT_S, -1000}, + + {"d", OPTION_UNIT_MIN, 60 * 24}, + {"h", OPTION_UNIT_MIN, 60}, + {"min", OPTION_UNIT_MIN, 1}, + {"s", OPTION_UNIT_MIN, -60}, + {"ms", OPTION_UNIT_MIN, -1000 * 60}, + + {""} /* end of table marker */ +}; + +static size_t +option_length(const pgut_option opts[]) +{ + size_t len; + + for (len = 0; opts && opts[len].type; len++) { } + + return len; +} + +static int +option_has_arg(char type) +{ + switch (type) + { + case 'b': + case 'B': + return no_argument; + default: + return required_argument; + } +} + +static void +option_copy(struct option dst[], const pgut_option opts[], size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) + { + dst[i].name = opts[i].lname; + dst[i].has_arg = option_has_arg(opts[i].type); + dst[i].flag = NULL; + dst[i].val = opts[i].sname; + } +} + +static pgut_option * +option_find(int c, pgut_option opts1[]) +{ + size_t i; + + for (i = 0; opts1 && opts1[i].type; i++) + if (opts1[i].sname == c) + return &opts1[i]; + + return NULL; /* not found */ +} + +static void +assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) +{ + const char *message; + + if (opt == NULL) + { + fprintf(stderr, "Try \"%s --help\" for more information.\n", PROGRAM_NAME); + exit_or_abort(ERROR); + } + + if (opt->source > src) + { + /* high prior value has been set already. */ + return; + } + /* Allow duplicate entries for function option */ + else if (src >= SOURCE_CMDLINE && opt->source >= src && opt->type != 'f') + { + message = "specified only once"; + } + else + { + pgut_optsrc orig_source = opt->source; + + /* can be overwritten if non-command line source */ + opt->source = src; + + switch (opt->type) + { + case 'b': + case 'B': + if (optarg == NULL) + { + *((bool *) opt->var) = (opt->type == 'b'); + return; + } + else if (parse_bool(optarg, (bool *) opt->var)) + { + return; + } + message = "a boolean"; + break; + case 'f': + ((pgut_optfn) opt->var)(opt, optarg); + return; + case 'i': + if (parse_int32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit signed integer"; + break; + case 'u': + if (parse_uint32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit unsigned integer"; + break; + case 'I': + if (parse_int64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit signed integer"; + break; + case 'U': + if (parse_uint64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit unsigned integer"; + break; + case 's': + if (orig_source != SOURCE_DEFAULT) + free(*(char **) opt->var); + *(char **) opt->var = pgut_strdup(optarg); + if (strcmp(optarg,"") != 0) + return; + message = "a valid string. But provided: "; + break; + case 't': + if (parse_time(optarg, opt->var, + opt->source == SOURCE_FILE)) + return; + message = "a time"; + break; + default: + elog(ERROR, "invalid option type: %c", opt->type); + return; /* keep compiler quiet */ + } + } + + if (isprint(opt->sname)) + elog(ERROR, "option -%c, --%s should be %s: '%s'", + opt->sname, opt->lname, message, optarg); + else + elog(ERROR, "option --%s should be %s: '%s'", + opt->lname, message, optarg); +} + +/* + * Convert a value from one of the human-friendly units ("kB", "min" etc.) + * to the given base unit. 'value' and 'unit' are the input value and unit + * to convert from. The converted value is stored in *base_value. + * + * Returns true on success, false if the input unit is not recognized. + */ +static bool +convert_to_base_unit(int64 value, const char *unit, + int base_unit, int64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Unsigned variant of convert_to_base_unit() + */ +static bool +convert_to_base_unit_u(uint64 value, const char *unit, + int base_unit, uint64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Convert a value in some base unit to a human-friendly unit. The output + * unit is chosen so that it's the greatest unit that can represent the value + * without loss. For example, if the base unit is GUC_UNIT_KB, 1024 is + * converted to 1 MB, but 1025 is represented as 1025 kB. + */ +void +convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +/* + * Unsigned variant of convert_from_base_unit() + */ +void +convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +static bool +parse_unit(char *unit_str, int flags, int64 value, int64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Unsigned variant of parse_unit() + */ +static bool +parse_unit_u(char *unit_str, int flags, uint64 value, uint64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit_u(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Try to interpret value as boolean value. Valid values are: true, + * false, yes, no, on, off, 1, 0; as well as unique prefixes thereof. + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + */ +bool +parse_bool(const char *value, bool *result) +{ + return parse_bool_with_len(value, strlen(value), result); +} + +bool +parse_bool_with_len(const char *value, size_t len, bool *result) +{ + switch (*value) + { + case 't': + case 'T': + if (pg_strncasecmp(value, "true", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'f': + case 'F': + if (pg_strncasecmp(value, "false", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'y': + case 'Y': + if (pg_strncasecmp(value, "yes", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'n': + case 'N': + if (pg_strncasecmp(value, "no", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'o': + case 'O': + /* 'o' is not unique enough */ + if (pg_strncasecmp(value, "on", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = true; + return true; + } + else if (pg_strncasecmp(value, "off", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = false; + return true; + } + break; + case '1': + if (len == 1) + { + if (result) + *result = true; + return true; + } + break; + case '0': + if (len == 1) + { + if (result) + *result = false; + return true; + } + break; + default: + break; + } + + if (result) + *result = false; /* suppress compiler warning */ + return false; +} + +/* + * Parse string as 32bit signed int. + * valid range: -2147483648 ~ 2147483647 + */ +bool +parse_int32(const char *value, int32 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = INT_MAX; + return true; + } + + errno = 0; + val = strtol(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE || val != (int64) ((int32) val)) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as 32bit unsigned int. + * valid range: 0 ~ 4294967295 (2^32-1) + */ +bool +parse_uint32(const char *value, uint32 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = UINT_MAX; + return true; + } + + errno = 0; + val = strtoul(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE || val != (uint64) ((uint32) val)) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as int64 + * valid range: -9223372036854775808 ~ 9223372036854775807 + */ +bool +parse_int64(const char *value, int64 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = LLONG_MAX; + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtol(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoll(value, &endptr, 0); +#else + val = strtol(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as uint64 + * valid range: 0 ~ (2^64-1) + */ +bool +parse_uint64(const char *value, uint64 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { +#if defined(HAVE_LONG_INT_64) + *result = ULONG_MAX; +#elif defined(HAVE_LONG_LONG_INT_64) + *result = ULLONG_MAX; +#else + *result = ULONG_MAX; +#endif + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtoul(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoull(value, &endptr, 0); +#else + val = strtoul(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Convert ISO-8601 format string to time_t value. + * + * If utc_default is true, then if timezone offset isn't specified tz will be + * +00:00. + */ +bool +parse_time(const char *value, time_t *result, bool utc_default) +{ + size_t len; + int fields_num, + tz = 0, + i; + bool tz_set = false; + char *tmp; + struct tm tm; + char junk[2]; + + /* tmp = replace( value, !isalnum, ' ' ) */ + tmp = pgut_malloc(strlen(value) + + 1); + len = 0; + fields_num = 1; + + while (*value) + { + if (IsAlnum(*value)) + { + tmp[len++] = *value; + value++; + } + else if (fields_num < 6) + { + fields_num++; + tmp[len++] = ' '; + value++; + } + /* timezone field is 7th */ + else if ((*value == '-' || *value == '+') && fields_num == 6) + { + int hr, + min, + sec = 0; + char *cp; + + errno = 0; + hr = strtol(value + 1, &cp, 10); + if ((value + 1) == cp || errno == ERANGE) + return false; + + /* explicit delimiter? */ + if (*cp == ':') + { + errno = 0; + min = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + if (*cp == ':') + { + errno = 0; + sec = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + } + } + /* otherwise, might have run things together... */ + else if (*cp == '\0' && strlen(value) > 3) + { + min = hr % 100; + hr = hr / 100; + /* we could, but don't, support a run-together hhmmss format */ + } + else + min = 0; + + /* Range-check the values; see notes in datatype/timestamp.h */ + if (hr < 0 || hr > MAX_TZDISP_HOUR) + return false; + if (min < 0 || min >= MINS_PER_HOUR) + return false; + if (sec < 0 || sec >= SECS_PER_MINUTE) + return false; + + tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; + if (*value == '-') + tz = -tz; + + tz_set = true; + + fields_num++; + value = cp; + } + /* wrong format */ + else if (!IsSpace(*value)) + return false; + } + tmp[len] = '\0'; + + /* parse for "YYYY-MM-DD HH:MI:SS" */ + memset(&tm, 0, sizeof(tm)); + tm.tm_year = 0; /* tm_year is year - 1900 */ + tm.tm_mon = 0; /* tm_mon is 0 - 11 */ + tm.tm_mday = 1; /* tm_mday is 1 - 31 */ + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); + free(tmp); + + if (i < 1 || 6 < i) + return false; + + /* adjust year */ + if (tm.tm_year < 100) + tm.tm_year += 2000 - 1900; + else if (tm.tm_year >= 1900) + tm.tm_year -= 1900; + + /* adjust month */ + if (i > 1) + tm.tm_mon -= 1; + + /* determine whether Daylight Saving Time is in effect */ + tm.tm_isdst = -1; + + *result = mktime(&tm); + + /* adjust time zone */ + if (tz_set || utc_default) + { + time_t ltime = time(NULL); + struct tm *ptm = gmtime(<ime); + time_t gmt = mktime(ptm); + time_t offset; + + /* UTC time */ + *result -= tz; + + /* Get local time */ + ptm = localtime(<ime); + offset = ltime - gmt + (ptm->tm_isdst ? 3600 : 0); + + *result += offset; + } + + return true; +} + +/* + * Try to parse value as an integer. The accepted formats are the + * usual decimal, octal, or hexadecimal formats, optionally followed by + * a unit name if "flags" indicates a unit is allowed. + * + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable + * HINT message, or NULL if no hint provided. + */ +bool +parse_int(const char *value, int *result, int flags, const char **hintmsg) +{ + int64 val; + char *endptr; + + /* To suppress compiler warnings, always set output params */ + if (result) + *result = 0; + if (hintmsg) + *hintmsg = NULL; + + /* We assume here that int64 is at least as wide as long */ + errno = 0; + val = strtol(value, &endptr, 0); + + if (endptr == value) + return false; /* no HINT for integer syntax error */ + + if (errno == ERANGE || val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + /* Handle possible unit */ + if (*endptr != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*endptr != '\0' && !isspace((unsigned char) *endptr) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(endptr++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + if (*endptr == '\0') + converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), + &val); + if (!converted) + { + /* invalid unit, or garbage after the unit; set hint and fail. */ + if (hintmsg) + { + if (flags & OPTION_UNIT_MEMORY) + *hintmsg = memory_units_hint; + else + *hintmsg = time_units_hint; + } + return false; + } + + /* Check for overflow due to units conversion */ + if (val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + } + + if (result) + *result = (int) val; + return true; +} + +bool +parse_lsn(const char *value, XLogRecPtr *result) +{ + uint32 xlogid; + uint32 xrecoff; + int len1; + int len2; + + len1 = strspn(value, "0123456789abcdefABCDEF"); + if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') + elog(ERROR, "invalid LSN \"%s\"", value); + len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); + if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') + elog(ERROR, "invalid LSN \"%s\"", value); + + if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) + *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + { + elog(ERROR, "invalid LSN \"%s\"", value); + return false; + } + + return true; +} + +static char * +longopts_to_optstring(const struct option opts[], const size_t len) +{ + size_t i; + char *result; + char *s; + + result = pgut_malloc(len * 2 + 1); + + s = result; + for (i = 0; i < len; i++) + { + if (!isprint(opts[i].val)) + continue; + *s++ = opts[i].val; + if (opts[i].has_arg != no_argument) + *s++ = ':'; + } + *s = '\0'; + + return result; +} + +void +pgut_getopt_env(pgut_option options[]) +{ + size_t i; + + for (i = 0; options && options[i].type; i++) + { + pgut_option *opt = &options[i]; + const char *value = NULL; + + /* If option was already set do not check env */ + if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV) + continue; + + if (strcmp(opt->lname, "pgdata") == 0) + value = getenv("PGDATA"); + if (strcmp(opt->lname, "port") == 0) + value = getenv("PGPORT"); + if (strcmp(opt->lname, "host") == 0) + value = getenv("PGHOST"); + if (strcmp(opt->lname, "username") == 0) + value = getenv("PGUSER"); + if (strcmp(opt->lname, "pgdatabase") == 0) + { + value = getenv("PGDATABASE"); + if (value == NULL) + value = getenv("PGUSER"); + if (value == NULL) + value = get_username(); + } + + if (value) + assign_option(opt, value, SOURCE_ENV); + } +} + +int +pgut_getopt(int argc, char **argv, pgut_option options[]) +{ + int c; + int optindex = 0; + char *optstring; + pgut_option *opt; + struct option *longopts; + size_t len; + + len = option_length(options); + longopts = pgut_newarray(struct option, len + 1 /* zero/end option */); + option_copy(longopts, options, len); + + optstring = longopts_to_optstring(longopts, len); + + /* Assign named options */ + while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) + { + opt = option_find(c, options); + if (opt && opt->allowed < SOURCE_CMDLINE) + elog(ERROR, "option %s cannot be specified in command line", + opt->lname); + /* Check 'opt == NULL' is performed in assign_option() */ + assign_option(opt, optarg, SOURCE_CMDLINE); + } + + init_cancel_handler(); + atexit(on_cleanup); + + return optind; +} + +/* compare two strings ignore cases and ignore -_ */ +static bool +key_equals(const char *lhs, const char *rhs) +{ + for (; *lhs && *rhs; lhs++, rhs++) + { + if (strchr("-_ ", *lhs)) + { + if (!strchr("-_ ", *rhs)) + return false; + } + else if (ToLower(*lhs) != ToLower(*rhs)) + return false; + } + + return *lhs == '\0' && *rhs == '\0'; +} + +/* + * Get configuration from configuration file. + * Return number of parsed options + */ +int +pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict) +{ + FILE *fp; + char buf[1024]; + char key[1024]; + char value[1024]; + int parsed_options = 0; + + if (!options) + return parsed_options; + + if ((fp = pgut_fopen(path, "rt", true)) == NULL) + return parsed_options; + + while (fgets(buf, lengthof(buf), fp)) + { + size_t i; + + for (i = strlen(buf); i > 0 && IsSpace(buf[i - 1]); i--) + buf[i - 1] = '\0'; + + if (parse_pair(buf, key, value)) + { + for (i = 0; options[i].type; i++) + { + pgut_option *opt = &options[i]; + + if (key_equals(key, opt->lname)) + { + if (opt->allowed < SOURCE_FILE && + opt->allowed != SOURCE_FILE_STRICT) + elog(elevel, "option %s cannot be specified in file", opt->lname); + else if (opt->source <= SOURCE_FILE) + { + assign_option(opt, value, SOURCE_FILE); + parsed_options++; + } + break; + } + } + if (strict && !options[i].type) + elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path); + } + } + + fclose(fp); + + return parsed_options; +} + +static const char * +skip_space(const char *str, const char *line) +{ + while (IsSpace(*str)) { str++; } + return str; +} + +static const char * +get_next_token(const char *src, char *dst, const char *line) +{ + const char *s; + int i; + int j; + + if ((s = skip_space(src, line)) == NULL) + return NULL; + + /* parse quoted string */ + if (*s == '\'') + { + s++; + for (i = 0, j = 0; s[i] != '\0'; i++) + { + if (s[i] == '\\') + { + i++; + switch (s[i]) + { + case 'b': + dst[j] = '\b'; + break; + case 'f': + dst[j] = '\f'; + break; + case 'n': + dst[j] = '\n'; + break; + case 'r': + dst[j] = '\r'; + break; + case 't': + dst[j] = '\t'; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { + int k; + long octVal = 0; + + for (k = 0; + s[i + k] >= '0' && s[i + k] <= '7' && k < 3; + k++) + octVal = (octVal << 3) + (s[i + k] - '0'); + i += k - 1; + dst[j] = ((char) octVal); + } + break; + default: + dst[j] = s[i]; + break; + } + } + else if (s[i] == '\'') + { + i++; + /* doubled quote becomes just one quote */ + if (s[i] == '\'') + dst[j] = s[i]; + else + break; + } + else + dst[j] = s[i]; + j++; + } + } + else + { + i = j = strcspn(s, "#\n\r\t\v"); + memcpy(dst, s, j); + } + + dst[j] = '\0'; + return s + i; +} + +static bool +parse_pair(const char buffer[], char key[], char value[]) +{ + const char *start; + const char *end; + + key[0] = value[0] = '\0'; + + /* + * parse key + */ + start = buffer; + if ((start = skip_space(start, buffer)) == NULL) + return false; + + end = start + strcspn(start, "=# \n\r\t\v"); + + /* skip blank buffer */ + if (end - start <= 0) + { + if (*start == '=') + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + /* key found */ + strncpy(key, start, end - start); + key[end - start] = '\0'; + + /* find key and value split char */ + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '=') + { + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + start++; + + /* + * parse value + */ + if ((end = get_next_token(start, value, buffer)) == NULL) + return false; + + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '\0' && *start != '#') + { + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + return true; +} + +/* + * Ask the user for a password; 'username' is the username the + * password is for, if one has been explicitly specified. + * Set malloc'd string to the global variable 'password'. + */ +static void +prompt_for_password(const char *username) +{ + in_password = true; + + if (password) + { + free(password); + password = NULL; + } + +#if PG_VERSION_NUM >= 100000 + password = (char *) pgut_malloc(sizeof(char) * 100 + 1); + if (username == NULL) + simple_prompt("Password: ", password, 100, false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + simple_prompt(message, password, 100, false); + } +#else + if (username == NULL) + password = simple_prompt("Password: ", 100, false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + password = simple_prompt(message, 100, false); + } +#endif + + in_password = false; +} + +/* + * Copied from pg_basebackup.c + * Escape a parameter value so that it can be used as part of a libpq + * connection string, e.g. in: + * + * application_name= + * + * The returned string is malloc'd. Return NULL on out-of-memory. + */ +static char * +escapeConnectionParameter(const char *src) +{ + bool need_quotes = false; + bool need_escaping = false; + const char *p; + char *dstbuf; + char *dst; + + /* + * First check if quoting is needed. Any quote (') or backslash (\) + * characters need to be escaped. Parameters are separated by whitespace, + * so any string containing whitespace characters need to be quoted. An + * empty string is represented by ''. + */ + if (strchr(src, '\'') != NULL || strchr(src, '\\') != NULL) + need_escaping = true; + + for (p = src; *p; p++) + { + if (isspace((unsigned char) *p)) + { + need_quotes = true; + break; + } + } + + if (*src == '\0') + return pg_strdup("''"); + + if (!need_quotes && !need_escaping) + return pg_strdup(src); /* no quoting or escaping needed */ + + /* + * Allocate a buffer large enough for the worst case that all the source + * characters need to be escaped, plus quotes. + */ + dstbuf = pg_malloc(strlen(src) * 2 + 2 + 1); + + dst = dstbuf; + if (need_quotes) + *(dst++) = '\''; + for (; *src; src++) + { + if (*src == '\'' || *src == '\\') + *(dst++) = '\\'; + *(dst++) = *src; + } + if (need_quotes) + *(dst++) = '\''; + *dst = '\0'; + + return dstbuf; +} + +/* Construct a connection string for possible future use in recovery.conf */ +char * +pgut_get_conninfo_string(PGconn *conn) +{ + PQconninfoOption *connOptions; + PQconninfoOption *option; + PQExpBuffer buf = createPQExpBuffer(); + char *connstr; + bool firstkeyword = true; + char *escaped; + + connOptions = PQconninfo(conn); + if (connOptions == NULL) + elog(ERROR, "out of memory"); + + /* Construct a new connection string in key='value' format. */ + for (option = connOptions; option && option->keyword; option++) + { + /* + * Do not emit this setting if: - the setting is "replication", + * "dbname" or "fallback_application_name", since these would be + * overridden by the libpqwalreceiver module anyway. - not set or + * empty. + */ + if (strcmp(option->keyword, "replication") == 0 || + strcmp(option->keyword, "dbname") == 0 || + strcmp(option->keyword, "fallback_application_name") == 0 || + (option->val == NULL) || + (option->val != NULL && option->val[0] == '\0')) + continue; + + /* do not print password into the file */ + if (strcmp(option->keyword, "password") == 0) + continue; + + if (!firstkeyword) + appendPQExpBufferChar(buf, ' '); + + firstkeyword = false; + + escaped = escapeConnectionParameter(option->val); + appendPQExpBuffer(buf, "%s=%s", option->keyword, escaped); + free(escaped); + } + + connstr = pg_strdup(buf->data); + destroyPQExpBuffer(buf); + return connstr; +} + +PGconn * +pgut_connect(const char *dbname) +{ + return pgut_connect_extended(host, port, dbname, username); +} + +PGconn * +pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login) +{ + PGconn *conn; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + if (force_password && !prompt_password) + elog(ERROR, "You cannot specify --password and --no-password options together"); + + if (!password && force_password) + prompt_for_password(login); + + /* Start the connection. Loop until we have a password if requested by backend. */ + for (;;) + { + conn = PQsetdbLogin(pghost, pgport, NULL, NULL, + dbname, login, password); + + if (PQstatus(conn) == CONNECTION_OK) + return conn; + + if (conn && PQconnectionNeedsPassword(conn) && prompt_password) + { + PQfinish(conn); + prompt_for_password(login); + + if (interrupted) + elog(ERROR, "interrupted"); + + if (password == NULL || password[0] == '\0') + elog(ERROR, "no password supplied"); + + continue; + } + elog(ERROR, "could not connect to database %s: %s", + dbname, PQerrorMessage(conn)); + + PQfinish(conn); + return NULL; + } +} + +PGconn * +pgut_connect_replication(const char *dbname) +{ + return pgut_connect_replication_extended(host, port, dbname, username); +} + +PGconn * +pgut_connect_replication_extended(const char *pghost, const char *pgport, + const char *dbname, const char *pguser) +{ + PGconn *tmpconn; + int argcount = 7; /* dbname, replication, fallback_app_name, + * host, user, port, password */ + int i; + const char **keywords; + const char **values; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + if (force_password && !prompt_password) + elog(ERROR, "You cannot specify --password and --no-password options together"); + + if (!password && force_password) + prompt_for_password(pguser); + + i = 0; + + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + + keywords[i] = "dbname"; + values[i] = "replication"; + i++; + keywords[i] = "replication"; + values[i] = "true"; + i++; + keywords[i] = "fallback_application_name"; + values[i] = PROGRAM_NAME; + i++; + + if (pghost) + { + keywords[i] = "host"; + values[i] = pghost; + i++; + } + if (pguser) + { + keywords[i] = "user"; + values[i] = pguser; + i++; + } + if (pgport) + { + keywords[i] = "port"; + values[i] = pgport; + i++; + } + + /* Use (or reuse, on a subsequent connection) password if we have it */ + if (password) + { + keywords[i] = "password"; + values[i] = password; + } + else + { + keywords[i] = NULL; + values[i] = NULL; + } + + for (;;) + { + tmpconn = PQconnectdbParams(keywords, values, true); + + + if (PQstatus(tmpconn) == CONNECTION_OK) + { + free(values); + free(keywords); + return tmpconn; + } + + if (tmpconn && PQconnectionNeedsPassword(tmpconn) && prompt_password) + { + PQfinish(tmpconn); + prompt_for_password(pguser); + keywords[i] = "password"; + values[i] = password; + continue; + } + + elog(ERROR, "could not connect to database %s: %s", + dbname, PQerrorMessage(tmpconn)); + PQfinish(tmpconn); + free(values); + free(keywords); + return NULL; + } +} + + +void +pgut_disconnect(PGconn *conn) +{ + if (conn) + PQfinish(conn); +} + +/* set/get host and port for connecting standby server */ +const char * +pgut_get_host() +{ + return host; +} + +const char * +pgut_get_port() +{ + return port; +} + +void +pgut_set_host(const char *new_host) +{ + host = new_host; +} + +void +pgut_set_port(const char *new_port) +{ + port = new_port; +} + + +PGresult * +pgut_execute_parallel(PGconn* conn, + PGcancel* thread_cancel_conn, const char *query, + int nParams, const char **params, + bool text_result) +{ + PGresult *res; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(ERROR, "not connected"); + return NULL; + } + + //on_before_exec(conn, thread_cancel_conn); + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + //on_after_exec(thread_cancel_conn); + + switch (PQresultStatus(res)) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + case PGRES_COPY_IN: + break; + default: + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + break; + } + + return res; +} + +PGresult * +pgut_execute(PGconn* conn, const char *query, int nParams, const char **params) +{ + return pgut_execute_extended(conn, query, nParams, params, true, false); +} + +PGresult * +pgut_execute_extended(PGconn* conn, const char *query, int nParams, + const char **params, bool text_result, bool ok_error) +{ + PGresult *res; + ExecStatusType res_status; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(ERROR, "not connected"); + return NULL; + } + + on_before_exec(conn, NULL); + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + on_after_exec(NULL); + + res_status = PQresultStatus(res); + switch (res_status) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + case PGRES_COPY_IN: + break; + default: + if (ok_error && res_status == PGRES_FATAL_ERROR) + break; + + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + break; + } + + return res; +} + +bool +pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel) +{ + int res; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(elevel, "not connected"); + return false; + } + + if (nParams == 0) + res = PQsendQuery(conn, query); + else + res = PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, 0); + + if (res != 1) + { + elog(elevel, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + return false; + } + + return true; +} + +void +pgut_cancel(PGconn* conn) +{ + PGcancel *cancel_conn = PQgetCancel(conn); + char errbuf[256]; + + if (cancel_conn != NULL) + { + if (PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + elog(WARNING, "Cancel request sent"); + else + elog(WARNING, "Cancel request failed"); + } + + if (cancel_conn) + PQfreeCancel(cancel_conn); +} + +int +pgut_wait(int num, PGconn *connections[], struct timeval *timeout) +{ + /* all connections are busy. wait for finish */ + while (!interrupted) + { + int i; + fd_set mask; + int maxsock; + + FD_ZERO(&mask); + + maxsock = -1; + for (i = 0; i < num; i++) + { + int sock; + + if (connections[i] == NULL) + continue; + sock = PQsocket(connections[i]); + if (sock >= 0) + { + FD_SET(sock, &mask); + if (maxsock < sock) + maxsock = sock; + } + } + + if (maxsock == -1) + { + errno = ENOENT; + return -1; + } + + i = wait_for_sockets(maxsock + 1, &mask, timeout); + if (i == 0) + break; /* timeout */ + + for (i = 0; i < num; i++) + { + if (connections[i] && FD_ISSET(PQsocket(connections[i]), &mask)) + { + PQconsumeInput(connections[i]); + if (PQisBusy(connections[i])) + continue; + return i; + } + } + } + + errno = EINTR; + return -1; +} + +#ifdef WIN32 +static CRITICAL_SECTION cancelConnLock; +#endif + +/* + * on_before_exec + * + * Set cancel_conn to point to the current database connection. + */ +static void +on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn) +{ + PGcancel *old; + + if (in_cleanup) + return; /* forbid cancel during cleanup */ + +#ifdef WIN32 + EnterCriticalSection(&cancelConnLock); +#endif + + if (thread_cancel_conn) + { + //elog(WARNING, "Handle tread_cancel_conn. on_before_exec"); + old = thread_cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + thread_cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + + thread_cancel_conn = PQgetCancel(conn); + } + else + { + /* Free the old one if we have one */ + old = cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + + cancel_conn = PQgetCancel(conn); + } + +#ifdef WIN32 + LeaveCriticalSection(&cancelConnLock); +#endif +} + +/* + * on_after_exec + * + * Free the current cancel connection, if any, and set to NULL. + */ +static void +on_after_exec(PGcancel *thread_cancel_conn) +{ + PGcancel *old; + + if (in_cleanup) + return; /* forbid cancel during cleanup */ + +#ifdef WIN32 + EnterCriticalSection(&cancelConnLock); +#endif + + if (thread_cancel_conn) + { + //elog(WARNING, "Handle tread_cancel_conn. on_after_exec"); + old = thread_cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + thread_cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + } + else + { + old = cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + } +#ifdef WIN32 + LeaveCriticalSection(&cancelConnLock); +#endif +} + +/* + * Handle interrupt signals by cancelling the current command. + */ +static void +on_interrupt(void) +{ + int save_errno = errno; + char errbuf[256]; + + /* Set interruped flag */ + interrupted = true; + + /* User promts password, call on_cleanup() byhand */ + if (in_password) + { + on_cleanup(); + + pqsignal(SIGINT, oldhandler); + kill(0, SIGINT); + } + + /* Send QueryCancel if we are processing a database query */ + if (!in_cleanup && cancel_conn != NULL && + PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + { + elog(WARNING, "Cancel request sent"); + } + + errno = save_errno; /* just in case the write changed it */ +} + +typedef struct pgut_atexit_item pgut_atexit_item; +struct pgut_atexit_item +{ + pgut_atexit_callback callback; + void *userdata; + pgut_atexit_item *next; +}; + +static pgut_atexit_item *pgut_atexit_stack = NULL; + +void +pgut_atexit_push(pgut_atexit_callback callback, void *userdata) +{ + pgut_atexit_item *item; + + AssertArg(callback != NULL); + + item = pgut_new(pgut_atexit_item); + item->callback = callback; + item->userdata = userdata; + item->next = pgut_atexit_stack; + + pgut_atexit_stack = item; +} + +void +pgut_atexit_pop(pgut_atexit_callback callback, void *userdata) +{ + pgut_atexit_item *item; + pgut_atexit_item **prev; + + for (item = pgut_atexit_stack, prev = &pgut_atexit_stack; + item; + prev = &item->next, item = item->next) + { + if (item->callback == callback && item->userdata == userdata) + { + *prev = item->next; + free(item); + break; + } + } +} + +static void +call_atexit_callbacks(bool fatal) +{ + pgut_atexit_item *item; + + for (item = pgut_atexit_stack; item; item = item->next) + item->callback(fatal, item->userdata); +} + +static void +on_cleanup(void) +{ + in_cleanup = true; + interrupted = false; + call_atexit_callbacks(false); +} + +static void +exit_or_abort(int exitcode) +{ + if (in_cleanup) + { + /* oops, error in cleanup*/ + call_atexit_callbacks(true); + abort(); + } + else + { + /* normal exit */ + exit(exitcode); + } +} + +/* + * Returns the current user name. + */ +static const char * +get_username(void) +{ + const char *ret; + +#ifndef WIN32 + struct passwd *pw; + + pw = getpwuid(geteuid()); + ret = (pw ? pw->pw_name : NULL); +#else + static char username[128]; /* remains after function exit */ + DWORD len = sizeof(username) - 1; + + if (GetUserName(username, &len)) + ret = username; + else + { + _dosmaperr(GetLastError()); + ret = NULL; + } +#endif + + if (ret == NULL) + elog(ERROR, "%s: could not get current user name: %s", + PROGRAM_NAME, strerror(errno)); + return ret; +} + +int +appendStringInfoFile(StringInfo str, FILE *fp) +{ + AssertArg(str != NULL); + AssertArg(fp != NULL); + + for (;;) + { + int rc; + + if (str->maxlen - str->len < 2 && enlargeStringInfo(str, 1024) == 0) + return errno = ENOMEM; + + rc = fread(str->data + str->len, 1, str->maxlen - str->len - 1, fp); + if (rc == 0) + break; + else if (rc > 0) + { + str->len += rc; + str->data[str->len] = '\0'; + } + else if (ferror(fp) && errno != EINTR) + return errno; + } + return 0; +} + +int +appendStringInfoFd(StringInfo str, int fd) +{ + AssertArg(str != NULL); + AssertArg(fd != -1); + + for (;;) + { + int rc; + + if (str->maxlen - str->len < 2 && enlargeStringInfo(str, 1024) == 0) + return errno = ENOMEM; + + rc = read(fd, str->data + str->len, str->maxlen - str->len - 1); + if (rc == 0) + break; + else if (rc > 0) + { + str->len += rc; + str->data[str->len] = '\0'; + } + else if (errno != EINTR) + return errno; + } + return 0; +} + +void * +pgut_malloc(size_t size) +{ + char *ret; + + if ((ret = malloc(size)) == NULL) + elog(ERROR, "could not allocate memory (%lu bytes): %s", + (unsigned long) size, strerror(errno)); + return ret; +} + +void * +pgut_realloc(void *p, size_t size) +{ + char *ret; + + if ((ret = realloc(p, size)) == NULL) + elog(ERROR, "could not re-allocate memory (%lu bytes): %s", + (unsigned long) size, strerror(errno)); + return ret; +} + +char * +pgut_strdup(const char *str) +{ + char *ret; + + if (str == NULL) + return NULL; + + if ((ret = strdup(str)) == NULL) + elog(ERROR, "could not duplicate string \"%s\": %s", + str, strerror(errno)); + return ret; +} + +char * +strdup_with_len(const char *str, size_t len) +{ + char *r; + + if (str == NULL) + return NULL; + + r = pgut_malloc(len + 1); + memcpy(r, str, len); + r[len] = '\0'; + return r; +} + +/* strdup but trim whitespaces at head and tail */ +char * +strdup_trim(const char *str) +{ + size_t len; + + if (str == NULL) + return NULL; + + while (IsSpace(str[0])) { str++; } + len = strlen(str); + while (len > 0 && IsSpace(str[len - 1])) { len--; } + + return strdup_with_len(str, len); +} + +FILE * +pgut_fopen(const char *path, const char *mode, bool missing_ok) +{ + FILE *fp; + + if ((fp = fopen(path, mode)) == NULL) + { + if (missing_ok && errno == ENOENT) + return NULL; + + elog(ERROR, "could not open file \"%s\": %s", + path, strerror(errno)); + } + + return fp; +} + +#ifdef WIN32 +static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout); +#define select select_win32 +#endif + +int +wait_for_socket(int sock, struct timeval *timeout) +{ + fd_set fds; + + FD_ZERO(&fds); + FD_SET(sock, &fds); + return wait_for_sockets(sock + 1, &fds, timeout); +} + +int +wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout) +{ + int i; + + for (;;) + { + i = select(nfds, fds, NULL, NULL, timeout); + if (i < 0) + { + if (interrupted) + elog(ERROR, "interrupted"); + else if (errno != EINTR) + elog(ERROR, "select failed: %s", strerror(errno)); + } + else + return i; + } +} + +#ifndef WIN32 +static void +handle_sigint(SIGNAL_ARGS) +{ + on_interrupt(); +} + +static void +init_cancel_handler(void) +{ + oldhandler = pqsignal(SIGINT, handle_sigint); +} +#else /* WIN32 */ + +/* + * Console control handler for Win32. Note that the control handler will + * execute on a *different thread* than the main one, so we need to do + * proper locking around those structures. + */ +static BOOL WINAPI +consoleHandler(DWORD dwCtrlType) +{ + if (dwCtrlType == CTRL_C_EVENT || + dwCtrlType == CTRL_BREAK_EVENT) + { + EnterCriticalSection(&cancelConnLock); + on_interrupt(); + LeaveCriticalSection(&cancelConnLock); + return TRUE; + } + else + /* Return FALSE for any signals not being handled */ + return FALSE; +} + +static void +init_cancel_handler(void) +{ + InitializeCriticalSection(&cancelConnLock); + + SetConsoleCtrlHandler(consoleHandler, TRUE); +} + +int +sleep(unsigned int seconds) +{ + Sleep(seconds * 1000); + return 0; +} + +int +usleep(unsigned int usec) +{ + Sleep((usec + 999) / 1000); /* rounded up */ + return 0; +} + +#undef select +static int +select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout) +{ + struct timeval remain; + + if (timeout != NULL) + remain = *timeout; + else + { + remain.tv_usec = 0; + remain.tv_sec = LONG_MAX; /* infinite */ + } + + /* sleep only one second because Ctrl+C doesn't interrupt select. */ + while (remain.tv_sec > 0 || remain.tv_usec > 0) + { + int ret; + struct timeval onesec; + + if (remain.tv_sec > 0) + { + onesec.tv_sec = 1; + onesec.tv_usec = 0; + remain.tv_sec -= 1; + } + else + { + onesec.tv_sec = 0; + onesec.tv_usec = remain.tv_usec; + remain.tv_usec = 0; + } + + ret = select(nfds, readfds, writefds, exceptfds, &onesec); + if (ret != 0) + { + /* succeeded or error */ + return ret; + } + else if (interrupted) + { + errno = EINTR; + return 0; + } + } + + return 0; /* timeout */ +} + +#endif /* WIN32 */ diff --git a/src/utils/pgut.h b/src/utils/pgut.h new file mode 100644 index 00000000..fedb99b0 --- /dev/null +++ b/src/utils/pgut.h @@ -0,0 +1,238 @@ +/*------------------------------------------------------------------------- + * + * pgut.h + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PGUT_H +#define PGUT_H + +#include "libpq-fe.h" +#include "pqexpbuffer.h" + +#include +#include + +#include "access/xlogdefs.h" +#include "logger.h" + +#if !defined(C_H) && !defined(__cplusplus) +#ifndef bool +typedef char bool; +#endif +#ifndef true +#define true ((bool) 1) +#endif +#ifndef false +#define false ((bool) 0) +#endif +#endif + +#define INFINITE_STR "INFINITE" + +typedef enum pgut_optsrc +{ + SOURCE_DEFAULT, + SOURCE_FILE_STRICT, + SOURCE_ENV, + SOURCE_FILE, + SOURCE_CMDLINE, + SOURCE_CONST +} pgut_optsrc; + +/* + * type: + * b: bool (true) + * B: bool (false) + * f: pgut_optfn + * i: 32bit signed integer + * u: 32bit unsigned integer + * I: 64bit signed integer + * U: 64bit unsigned integer + * s: string + * t: time_t + */ +typedef struct pgut_option +{ + char type; + uint8 sname; /* short name */ + const char *lname; /* long name */ + void *var; /* pointer to variable */ + pgut_optsrc allowed; /* allowed source */ + pgut_optsrc source; /* actual source */ + int flags; /* option unit */ +} pgut_option; + +typedef void (*pgut_optfn) (pgut_option *opt, const char *arg); +typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); + +/* + * bit values in "flags" of an option + */ +#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ +#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ +#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ +#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ +#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ + +#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ +#define OPTION_UNIT_S 0x20000 /* value is in seconds */ +#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ +#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ + +#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) + +/* + * pgut client variables and functions + */ +extern const char *PROGRAM_NAME; +extern const char *PROGRAM_VERSION; +extern const char *PROGRAM_URL; +extern const char *PROGRAM_EMAIL; + +extern void pgut_help(bool details); + +/* + * pgut framework variables and functions + */ +extern const char *pgut_dbname; +extern const char *host; +extern const char *port; +extern const char *username; +extern bool prompt_password; +extern bool force_password; + +extern bool interrupted; +extern bool in_cleanup; +extern bool in_password; /* User prompts password */ + +extern int pgut_getopt(int argc, char **argv, pgut_option options[]); +extern int pgut_readopt(const char *path, pgut_option options[], int elevel, + bool strict); +extern void pgut_getopt_env(pgut_option options[]); +extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); +extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); + +/* + * Database connections + */ +extern char *pgut_get_conninfo_string(PGconn *conn); +extern PGconn *pgut_connect(const char *dbname); +extern PGconn *pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login); +extern PGconn *pgut_connect_replication(const char *dbname); +extern PGconn *pgut_connect_replication_extended(const char *pghost, const char *pgport, + const char *dbname, const char *pguser); +extern void pgut_disconnect(PGconn *conn); +extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, + const char **params); +extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nParams, + const char **params, bool text_result, bool ok_error); +extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, + const char *query, int nParams, + const char **params, bool text_result); +extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); +extern void pgut_cancel(PGconn* conn); +extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); + +extern const char *pgut_get_host(void); +extern const char *pgut_get_port(void); +extern void pgut_set_host(const char *new_host); +extern void pgut_set_port(const char *new_port); + +/* + * memory allocators + */ +extern void *pgut_malloc(size_t size); +extern void *pgut_realloc(void *p, size_t size); +extern char *pgut_strdup(const char *str); +extern char *strdup_with_len(const char *str, size_t len); +extern char *strdup_trim(const char *str); + +#define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) +#define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) + +/* + * file operations + */ +extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); + +/* + * Assert + */ +#undef Assert +#undef AssertArg +#undef AssertMacro + +#ifdef USE_ASSERT_CHECKING +#define Assert(x) assert(x) +#define AssertArg(x) assert(x) +#define AssertMacro(x) assert(x) +#else +#define Assert(x) ((void) 0) +#define AssertArg(x) ((void) 0) +#define AssertMacro(x) ((void) 0) +#endif + +/* + * StringInfo and string operations + */ +#define STRINGINFO_H + +#define StringInfoData PQExpBufferData +#define StringInfo PQExpBuffer +#define makeStringInfo createPQExpBuffer +#define initStringInfo initPQExpBuffer +#define freeStringInfo destroyPQExpBuffer +#define termStringInfo termPQExpBuffer +#define resetStringInfo resetPQExpBuffer +#define enlargeStringInfo enlargePQExpBuffer +#define printfStringInfo printfPQExpBuffer /* reset + append */ +#define appendStringInfo appendPQExpBuffer +#define appendStringInfoString appendPQExpBufferStr +#define appendStringInfoChar appendPQExpBufferChar +#define appendBinaryStringInfo appendBinaryPQExpBuffer + +extern int appendStringInfoFile(StringInfo str, FILE *fp); +extern int appendStringInfoFd(StringInfo str, int fd); + +extern bool parse_bool(const char *value, bool *result); +extern bool parse_bool_with_len(const char *value, size_t len, bool *result); +extern bool parse_int32(const char *value, int32 *result, int flags); +extern bool parse_uint32(const char *value, uint32 *result, int flags); +extern bool parse_int64(const char *value, int64 *result, int flags); +extern bool parse_uint64(const char *value, uint64 *result, int flags); +extern bool parse_time(const char *value, time_t *result, bool utc_default); +extern bool parse_int(const char *value, int *result, int flags, + const char **hintmsg); +extern bool parse_lsn(const char *value, XLogRecPtr *result); + +extern void convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit); +extern void convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit); + +#define IsSpace(c) (isspace((unsigned char)(c))) +#define IsAlpha(c) (isalpha((unsigned char)(c))) +#define IsAlnum(c) (isalnum((unsigned char)(c))) +#define IsIdentHead(c) (IsAlpha(c) || (c) == '_') +#define IsIdentBody(c) (IsAlnum(c) || (c) == '_') +#define ToLower(c) (tolower((unsigned char)(c))) +#define ToUpper(c) (toupper((unsigned char)(c))) + +/* + * socket operations + */ +extern int wait_for_socket(int sock, struct timeval *timeout); +extern int wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout); + +#ifdef WIN32 +extern int sleep(unsigned int seconds); +extern int usleep(unsigned int usec); +#endif + +#endif /* PGUT_H */ diff --git a/src/utils/thread.c b/src/utils/thread.c new file mode 100644 index 00000000..82c23764 --- /dev/null +++ b/src/utils/thread.c @@ -0,0 +1,102 @@ +/*------------------------------------------------------------------------- + * + * thread.c: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "thread.h" + +pthread_t main_tid = 0; + +#ifdef WIN32 +#include + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} win32_pthread; + +static long mutex_initlock = 0; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + + th->result = th->routine(th->arg); + + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ + +int +pthread_lock(pthread_mutex_t *mp) +{ +#ifdef WIN32 + if (*mp == NULL) + { + while (InterlockedExchange(&mutex_initlock, 1) == 1) + /* loop, another thread own the lock */ ; + if (*mp == NULL) + { + if (pthread_mutex_init(mp, NULL)) + return -1; + } + InterlockedExchange(&mutex_initlock, 0); + } +#endif + return pthread_mutex_lock(mp); +} diff --git a/src/utils/thread.h b/src/utils/thread.h new file mode 100644 index 00000000..06460533 --- /dev/null +++ b/src/utils/thread.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * thread.h: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_THREAD_H +#define PROBACKUP_THREAD_H + +#ifdef WIN32 +#include "postgres_fe.h" +#include "port/pthread-win32.h" + +/* Use native win32 threads on Windows */ +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; + +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +extern int pthread_join(pthread_t th, void **thread_return); +#else +/* Use platform-dependent pthread capability */ +#include +#endif + +extern pthread_t main_tid; + +extern int pthread_lock(pthread_mutex_t *mp); + +#endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c new file mode 100644 index 00000000..bc82e811 --- /dev/null +++ b/src/validate.c @@ -0,0 +1,354 @@ +/*------------------------------------------------------------------------- + * + * validate.c: validate backup files. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); +static void do_validate_instance(void); + +static bool corrupted_backup_found = false; + +typedef struct +{ + parray *files; + bool corrupted; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} validate_files_arg; + +/* + * Validate backup files. + */ +void +pgBackupValidate(pgBackup *backup) +{ + char base_path[MAXPGPATH]; + char path[MAXPGPATH]; + parray *files; + bool corrupted = false; + bool validation_isok = true; + /* arrays with meta info for multi threaded validate */ + pthread_t *threads; + validate_files_arg *threads_args; + int i; + + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE && + backup->status != BACKUP_STATUS_ORPHAN && + backup->status != BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s has status %s. Skip validation.", + base36enc(backup->start_time), status2str(backup->status)); + corrupted_backup_found = true; + return; + } + + if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) + elog(INFO, "Validating backup %s", base36enc(backup->start_time)); + else + elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); + + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->backup_mode != BACKUP_MODE_DIFF_PAGE && + backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && + backup->backup_mode != BACKUP_MODE_DIFF_DELTA) + elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); + + pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + files = dir_read_file_list(base_path, path); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + pg_atomic_clear_flag(&file->lock); + } + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (validate_files_arg *) + palloc(sizeof(validate_files_arg) * num_threads); + + /* Validate files */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->corrupted = false; + /* By default there are some error */ + threads_args[i].ret = 1; + + pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + pthread_join(threads[i], NULL); + if (arg->corrupted) + corrupted = true; + if (arg->ret == 1) + validation_isok = false; + } + if (!validation_isok) + elog(ERROR, "Data files validation failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + /* Update backup status */ + backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; + pgBackupWriteBackupControlFile(backup); + + if (corrupted) + elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); + else + elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); +} + +/* + * Validate files in the backup. + * NOTE: If file is not valid, do not use ERROR log message, + * rather throw a WARNING and set arguments->corrupted = true. + * This is necessary to update backup status. + */ +static void * +pgBackupValidateFiles(void *arg) +{ + int i; + validate_files_arg *arguments = (validate_files_arg *)arg; + pg_crc32 crc; + + for (i = 0; i < parray_num(arguments->files); i++) + { + struct stat st; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + if (interrupted) + elog(ERROR, "Interrupted during validate"); + + /* Validate only regular files */ + if (!S_ISREG(file->mode)) + continue; + /* + * Skip files which has no data, because they + * haven't changed between backups. + */ + if (file->write_size == BYTES_INVALID) + continue; + + /* + * Currently we don't compute checksums for + * cfs_compressed data files, so skip them. + */ + if (file->is_cfs) + continue; + + /* print progress */ + elog(VERBOSE, "Validate files: (%d/%lu) %s", + i + 1, (unsigned long) parray_num(arguments->files), file->path); + + if (stat(file->path, &st) == -1) + { + if (errno == ENOENT) + elog(WARNING, "Backup file \"%s\" is not found", file->path); + else + elog(WARNING, "Cannot stat backup file \"%s\": %s", + file->path, strerror(errno)); + arguments->corrupted = true; + break; + } + + if (file->write_size != st.st_size) + { + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); + arguments->corrupted = true; + break; + } + + crc = pgFileGetCRC(file->path); + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + arguments->corrupted = true; + break; + } + } + + /* Data files validation is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Validate all backups in the backup catalog. + * If --instance option was provided, validate only backups of this instance. + */ +int +do_validate_all(void) +{ + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + xlog_seg_size = get_config_xlog_seg_size(); + + do_validate_instance(); + } + } + else + { + do_validate_instance(); + } + + if (corrupted_backup_found) + { + elog(WARNING, "Some backups are not valid"); + return 1; + } + else + elog(INFO, "All backups are valid"); + + return 0; +} + +/* + * Validate all backups in the given instance of the backup catalog. + */ +static void +do_validate_instance(void) +{ + char *current_backup_id; + int i; + parray *backups; + pgBackup *current_backup = NULL; + + elog(INFO, "Validate backups of the instance '%s'", instance_name); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Examine backups one by one and validate them */ + for (i = 0; i < parray_num(backups); i++) + { + current_backup = (pgBackup *) parray_get(backups, i); + + /* Valiate each backup along with its xlog files. */ + pgBackupValidate(current_backup); + + /* Ensure that the backup has valid list of parent backups */ + if (current_backup->status == BACKUP_STATUS_OK) + { + pgBackup *base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* Validate corresponding WAL files */ + validate_wal(current_backup, arclog_path, 0, + 0, 0, base_full_backup->tli, xlog_seg_size); + } + + /* Mark every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (current_backup->status == BACKUP_STATUS_CORRUPT) + { + int j; + + corrupted_backup_found = true; + current_backup_id = base36enc_dup(current_backup->start_time); + for (j = i - 1; j >= 0; j--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, j); + + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + base36enc(backup->start_time), current_backup_id); + } + } + free(current_backup_id); + } + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); +} diff --git a/tests/Readme.md b/tests/Readme.md new file mode 100644 index 00000000..31dfb656 --- /dev/null +++ b/tests/Readme.md @@ -0,0 +1,24 @@ +[см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) + +``` +Note: For now there are tests only for Linix +``` + + +``` +Check physical correctness of restored instances: + Apply this patch to disable HINT BITS: https://gist.github.com/gsmol/2bb34fd3ba31984369a72cc1c27a36b6 + export PG_PROBACKUP_PARANOIA=ON + +Check archive compression: + export ARCHIVE_COMPRESSION=ON + +Specify path to pg_probackup binary file. By default tests use /pg_probackup/ + export PGPROBACKUPBIN= + +Usage: + pip install testgres + pip install psycopg2 + export PG_CONFIG=/path/to/pg_config + python -m unittest [-v] tests[.specific_module][.class.test] +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..aeeabf2a --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,69 @@ +import unittest + +from . import init_test, option_test, show_test, \ + backup_test, delete_test, restore_test, validate_test, \ + retention_test, ptrack_clean, ptrack_cluster, \ + ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ + ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ + ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ + false_positive, replica, compression, page, ptrack, archive, \ + exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test + + +def load_tests(loader, tests, pattern): + suite = unittest.TestSuite() +# suite.addTests(loader.loadTestsFromModule(auth_test)) + suite.addTests(loader.loadTestsFromModule(archive)) + suite.addTests(loader.loadTestsFromModule(backup_test)) + suite.addTests(loader.loadTestsFromModule(cfs_backup)) +# suite.addTests(loader.loadTestsFromModule(cfs_restore)) +# suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) +# suite.addTests(loader.loadTestsFromModule(logging)) + suite.addTests(loader.loadTestsFromModule(compression)) + suite.addTests(loader.loadTestsFromModule(delete_test)) + suite.addTests(loader.loadTestsFromModule(exclude)) + suite.addTests(loader.loadTestsFromModule(false_positive)) + suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(option_test)) + suite.addTests(loader.loadTestsFromModule(page)) + suite.addTests(loader.loadTestsFromModule(ptrack)) + suite.addTests(loader.loadTestsFromModule(ptrack_clean)) + suite.addTests(loader.loadTestsFromModule(ptrack_cluster)) + suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace)) + suite.addTests(loader.loadTestsFromModule(ptrack_recovery)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) + suite.addTests(loader.loadTestsFromModule(replica)) + suite.addTests(loader.loadTestsFromModule(restore_test)) + suite.addTests(loader.loadTestsFromModule(retention_test)) + suite.addTests(loader.loadTestsFromModule(show_test)) + suite.addTests(loader.loadTestsFromModule(validate_test)) + suite.addTests(loader.loadTestsFromModule(pgpro560)) + suite.addTests(loader.loadTestsFromModule(pgpro589)) + + return suite + +# test_pgpro434_2 unexpected success +# ToDo: +# archive: +# discrepancy of instance`s SYSTEMID and node`s SYSTEMID should lead to archive-push refusal to work +# replica: +# backup should exit with correct error message if some master* option is missing +# --master* options shoukd not work when backuping master +# logging: +# https://jira.postgrespro.ru/browse/PGPRO-584 +# https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md +# ptrack: +# ptrack backup on replica should work correctly +# archive: +# immediate recovery and full recovery +# backward compatibility: +# previous version catalog must be readable by newer version +# incremental chain from previous version can be continued +# backups from previous version can be restored +# 10vanilla_1.3ptrack + +# 10vanilla+ +# 9.6vanilla_1.3ptrack + diff --git a/tests/archive.py b/tests/archive.py new file mode 100644 index 00000000..8b8eb71a --- /dev/null +++ b/tests/archive.py @@ -0,0 +1,833 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script +from datetime import datetime, timedelta +import subprocess +from sys import exit +from time import sleep + + +module_name = 'archive' + + +class ArchiveTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_pgpro434_1(self): + """Description in jira issue PGPRO-434""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector from " + "generate_series(0,100) i") + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-file=verbose"]) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node) + node.slow_start() + + # Recreate backup calagoue + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Make backup + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-file=verbose"]) + node.cleanup() + + # Restore Database + self.restore_node( + backup_dir, 'node', node, + options=["--recovery-target-action=promote"]) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro434_2(self): + """ + Check that timelines are correct. + WAITING PGPRO-1053 for --immediate + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FIRST TIMELINE + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + backup_id = self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select 100501 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + # SECOND TIMELIN + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 " + "from t_heap where id = 100501)")[0][0], + 'data after restore not equal to original data') + + node.safe_psql( + "postgres", + "insert into t_heap select 2 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(100,200) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select 100502 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # THIRD TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print( + node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + node.safe_psql( + "postgres", + "insert into t_heap select 3 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(200,300) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.safe_psql( + "postgres", + "insert into t_heap select 100503 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # FOURTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fourth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # FIFTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fifth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # SIXTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Sixth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 from t_heap where id > 100500)")[0][0], + 'data after restore not equal to original data') + + self.assertEqual( + result, + node.safe_psql( + "postgres", + "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro434_3(self): + """Check pg_stop_backup_timeout, needed backup_timeout""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + archive_script_path = os.path.join(backup_dir, 'archive_script.sh') + with open(archive_script_path, 'w+') as f: + f.write( + archive_script.format( + backup_dir=backup_dir, node_name='node', count_limit=2)) + + st = os.stat(archive_script_path) + os.chmod(archive_script_path, st.st_mode | 0o111) + node.append_conf( + 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( + archive_script_path)) + node.slow_start() + try: + self.backup_node( + backup_dir, 'node', node, + options=[ + "--archive-timeout=60", + "--log-level-file=verbose", + "--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because pg_stop_backup failed to answer.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: pg_stop_backup doesn't answer" in e.message and + "cancel it" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_arhive_push_file_exists(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + file = os.path.join(wals_dir, '000000010000000000000001.gz') + else: + file = os.path.join(wals_dir, '000000010000000000000001') + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content and + 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + os.remove(file) + self.switch_wal_segment(node) + sleep(5) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_arhive_push_file_exists_overwrite(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + file = os.path.join(wals_dir, '000000010000000000000001.gz') + else: + file = os.path.join(wals_dir, '000000010000000000000001') + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content and + 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + self.set_archiving(backup_dir, 'node', node, overwrite=True) + node.reload() + self.switch_wal_segment(node) + sleep(2) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_replica_archive(self): + """ + make node without archiving, take stream backup and + turn it into replica, set replica with archiving, + make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'max_wal_size': '1GB'} + ) + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica, synchronous=True) + + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # ADD INSTANCE 'REPLICA' + + sleep(1) + + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', + replica, backup_type='page', + options=[ + '--archive-timeout=30', '--log-level-file=verbose', + '--master-host=localhost', '--master-db=postgres', + '--master-port={0}'.format(master.port)] + ) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_parallel_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=True) + # ADD INSTANCE REPLICA + self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=20', + '--log-level-file=verbose', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)] + ) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_concurrent_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node( + backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=True) + # ADD INSTANCE REPLICA + # self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + # self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node( + backup_dir, 'master', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog(self): + """Test backup with pg_receivexlog wal delivary method""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + if self.get_version(node) < 100000: + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node') + ], async=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, + node.safe_psql( + "postgres", "SELECT * FROM t_heap" + ), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog_compression_pg10(self): + """Test backup with pg_receivewal compressed wal delivary method""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + if self.get_version(node) < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL 10 for this test') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') + ], async=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, 'node', node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) diff --git a/tests/auth_test.py b/tests/auth_test.py new file mode 100644 index 00000000..fc21a480 --- /dev/null +++ b/tests/auth_test.py @@ -0,0 +1,391 @@ +""" +The Test suite check behavior of pg_probackup utility, if password is required for connection to PostgreSQL instance. + - https://confluence.postgrespro.ru/pages/viewpage.action?pageId=16777522 +""" + +import os +import unittest +import signal +import time + +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import StartNodeException + +module_name = 'auth_test' +skip_test = False + + +try: + from pexpect import * +except ImportError: + skip_test = True + + +class SimpleAuthTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_backup_via_unpriviledged_user(self): + """ + Make node, create unpriviledged user, try to + run a backups without EXECUTE rights on + certain functions + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_start_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_start_backup(text, boolean, boolean) TO backup;") + + time.sleep(1) + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied for function " + "pg_create_restore_point\nquery was: " + "SELECT pg_catalog.pg_create_restore_point($1)", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_create_restore_point(text) TO backup;") + + time.sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_stop_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + if self.get_version(node) < self.version_to_num('10.0'): + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION " + "pg_stop_backup(boolean, boolean) TO backup") + # Do this for ptrack backups + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup") + + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + + node.safe_psql("postgres", "CREATE DATABASE test1") + + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + + node.safe_psql( + "test1", "create table t1 as select generate_series(0,100)") + + node.append_conf("postgresql.auto.conf", "ptrack_enable = 'on'") + node.restart() + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on clearing ptrack_files.") + except ProbackupException as e: + self.assertIn( + "ERROR: must be superuser or replication role to clear ptrack files\n" + "query was: SELECT pg_catalog.pg_ptrack_clear()", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + time.sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on clearing ptrack_files.") + except ProbackupException as e: + self.assertIn( + "ERROR: must be superuser or replication role read ptrack files\n" + "query was: select pg_catalog.pg_ptrack_control_lsn()", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "ALTER ROLE backup REPLICATION") + + time.sleep(1) + + # FULL + self.backup_node( + backup_dir, 'node', node, + options=['-U', 'backup']) + + # PTRACK + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + +class AuthTest(unittest.TestCase): + pb = None + node = None + + @classmethod + def setUpClass(cls): + + super(AuthTest, cls).setUpClass() + + cls.pb = ProbackupTest() + cls.backup_dir = os.path.join(cls.pb.tmp_path, module_name, 'backup') + + cls.node = cls.pb.make_simple_node( + base_dir="{}/node".format(module_name), + set_replication=True, + initdb_params=['--data-checksums', '--auth-host=md5'], + pg_options={ + 'wal_level': 'replica' + } + ) + modify_pg_hba(cls.node) + + cls.pb.init_pb(cls.backup_dir) + cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) + cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) + try: + cls.node.start() + except StartNodeException: + raise unittest.skip("Node hasn't started") + + cls.node.safe_psql("postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; \ + GRANT USAGE ON SCHEMA pg_catalog TO backup; \ + GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_current() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") + cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + + @classmethod + def tearDownClass(cls): + cls.node.cleanup() + cls.pb.del_test_dir(module_name, '') + + @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") + def setUp(self): + self.cmd = ['backup', + '-B', self.backup_dir, + '--instance', self.node.name, + '-h', '127.0.0.1', + '-p', str(self.node.port), + '-U', 'backup', + '-b', 'FULL' + ] + + def tearDown(self): + if "PGPASSWORD" in self.pb.test_env.keys(): + del self.pb.test_env["PGPASSWORD"] + + if "PGPASSWORD" in self.pb.test_env.keys(): + del self.pb.test_env["PGPASSFILE"] + + try: + os.remove(self.pgpass_file) + except OSError: + pass + + def test_empty_password(self): + """ Test case: PGPB_AUTH03 - zero password length """ + try: + self.assertIn("ERROR: no password supplied", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, '\0\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_wrong_password(self): + """ Test case: PGPB_AUTH04 - incorrect password """ + try: + self.assertIn("password authentication failed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'wrong_password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_right_password(self): + """ Test case: PGPB_AUTH01 - correct password """ + try: + self.assertIn("completed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_right_password_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + create_pgpass(self.pgpass_file, line) + try: + self.assertIn("completed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd + ['-W'], 'password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_ctrl_c_event(self): + """ Test case: PGPB_AUTH02 - send interrupt signal """ + try: + run_pb_with_auth([self.pb.probackup_path] + self.cmd, kill=True) + except TIMEOUT: + self.fail("Error: CTRL+C event ignored") + + def test_pgpassfile_env(self): + """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ + path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + create_pgpass(path, line) + self.pb.test_env["PGPASSFILE"] = path + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpass(self): + """ Test case: PGPB_AUTH07 - Create file .pgpass in home dir. """ + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + create_pgpass(self.pgpass_file, line) + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpassword(self): + """ Test case: PGPB_AUTH08 - set environment var PGPASSWORD """ + self.pb.test_env["PGPASSWORD"] = "password" + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpassword_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH09 - Check priority between PGPASSWORD and .pgpass file""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + create_pgpass(self.pgpass_file, line) + self.pb.test_env["PGPASSWORD"] = "password" + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + +def run_pb_with_auth(cmd, password=None, kill=False): + try: + with spawn(" ".join(cmd), encoding='utf-8', timeout=10) as probackup: + result = probackup.expect(u"Password for user .*:", 5) + if kill: + probackup.kill(signal.SIGINT) + elif result == 0: + probackup.sendline(password) + probackup.expect(EOF) + return probackup.before + else: + raise ExceptionPexpect("Other pexpect errors.") + except TIMEOUT: + raise TIMEOUT("Timeout error.") + except ExceptionPexpect: + raise ExceptionPexpect("Pexpect error.") + + +def modify_pg_hba(node): + """ + Description: + Add trust authentication for user postgres. Need for add new role and set grant. + :param node: + :return None: + """ + hba_conf = os.path.join(node.data_dir, "pg_hba.conf") + with open(hba_conf, 'r+') as fio: + data = fio.read() + fio.seek(0) + fio.write('host\tall\tpostgres\t127.0.0.1/0\ttrust\n' + data) + + +def create_pgpass(path, line): + with open(path, 'w') as passfile: + # host:port:db:username:password + passfile.write(line) + os.chmod(path, 0o600) diff --git a/tests/backup_test.py b/tests/backup_test.py new file mode 100644 index 00000000..1fa74643 --- /dev/null +++ b/tests/backup_test.py @@ -0,0 +1,522 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.cfs_helpers import find_by_name + + +module_name = 'backup' + + +class BackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-707 + def test_backup_modes_archive(self): + """standart backup modes with ARCHIVE WAL method""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # postmaster.pid and postmaster.opts shouldn't be copied + excluded = True + db_dir = os.path.join( + backup_dir, "backups", 'node', backup_id, "database") + + for f in os.listdir(db_dir): + if ( + os.path.isfile(os.path.join(db_dir, f)) and + ( + f == "postmaster.pid" or + f == "postmaster.opts" + ) + ): + excluded = False + self.assertEqual(excluded, True) + + # page backup mode + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + # print self.show_pb(node) + show_backup = self.show_pb(backup_dir, 'node')[1] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Check parent backup + self.assertEqual( + backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup['id'])["parent-backup-id"]) + + # ptrack backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + show_backup = self.show_pb(backup_dir, 'node')[2] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PTRACK") + + # Check parent backup + self.assertEqual( + page_backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup['id'])["parent-backup-id"]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_smooth_checkpoint(self): + """full backup with smooth checkpoint""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + options=["-C"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + node.stop() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incremental_backup_without_full(self): + """page-level backup without validated full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], + "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incremental_backup_corrupt_full(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", backup_id, + "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of validation of corrupted backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'\n" in e.message and + "WARNING: Backup file \"{0}\" is not found\n".format( + file) in e.message and + "WARNING: Backup {0} data files are corrupted\n".format( + backup_id) in e.message and + "WARNING: Some backups are not valid\n" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_threads(self): + """ptrack multi thread backup mode""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_threads_stream(self): + """ptrack multi thread backup mode and stream""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_1(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream", '--log-level-file=verbose']) + + # open log file and check + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) + f.close + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', + "Backup Status should be OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_2(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of page " + "corruption in PostgreSQL instance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', + "Backup Status should be ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_tablespace_in_pgdata_pgpro_1376(self): + """PGPRO-1376 """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=( + os.path.join( + node.data_dir, 'somedirectory', '100500')) + ) + + self.create_tblspace_in_node( + node, 'tblspace2', + tblspc_path=(os.path.join(node.data_dir)) + ) + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace tblspace1 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + node.safe_psql( + "postgres", + "create table t_heap2 tablespace tblspace2 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of too many levels " + "of symbolic linking\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Too many levels of symbolic links' in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "drop table t_heap2") + node.safe_psql( + "postgres", + "drop tablespace tblspace2") + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content(node.data_dir) + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap1'::regclass::oid" + ).rstrip() + + list = [] + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file == relfilenode: + path = os.path.join(root, file) + list = list + [path] + + # We expect that relfilenode occures only once + if len(list) > 1: + message = "" + for string in list: + message = message + string + "\n" + self.assertEqual( + 1, 0, + "Following file copied twice by backup:\n {0}".format( + message) + ) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py new file mode 100644 index 00000000..41232032 --- /dev/null +++ b/tests/cfs_backup.py @@ -0,0 +1,1161 @@ +import os +import unittest +import random +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): + # --- Begin --- # + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def setUp(self): + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join( + self.tmp_path, module_name, self.fname, 'backup') + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2', + 'shared_buffers': '200MB' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.start() + + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( + tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not created " + "or it create without compressions" + ) + + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + + # --- Section: Full --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace(self): + """Case: Check fullbackup empty compressed tablespace""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_stream(self): + """Case: Check fullbackup empty compressed tablespace with options stream""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table(self): + """Case: Make full backup after created table in the tablespace""" + if not self.enterprise: + return + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "\n ERROR: {0}\n CMD: {1}".format( + repr(e.message), + repr(self.cmd) + ) + ) + return False + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in {0}".format( + os.path.join(self.backup_dir, 'node', backup_id)) + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table_stream(self): + """ + Case: Make full backup after created table in the tablespace with option --stream + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # --- Section: Incremental from empty tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # --- Section: Incremental from fill tablespace --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,25) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap', tblspace_name) + ) + + full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap') + ) + + page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, options=["-j", "4"]) + self.node.start() + self.assertEqual( + full_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, options=["-j", "4"]) + self.node.start() + self.assertEqual( + page_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments_in_multiple_tablespaces(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + tblspace_name_1 = 'tblspace_name_1' + tblspace_name_2 = 'tblspace_name_2' + + self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True) + self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_1', tblspace_name_1) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_2', tblspace_name_2) + ) + + full_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + full_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_1') + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_2') + ) + + page_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + page_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_1), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_2), + ignore_errors=True) + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, options=["-j", "4"]) + self.node.start() + self.assertEqual( + full_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + full_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_1), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_2), + ignore_errors=True) + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, options=["-j", "4"]) + self.node.start() + self.assertEqual( + page_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + page_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # --- Make backup with not valid data(broken .cfm) --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_cfm_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + os.remove(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_file_pg_compression_from_tablespace_dir(self): + os.remove( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0]) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_data_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + os.remove(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_cfm_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + corrupt_file(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_data_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + corrupt_file(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_file_pg_compression_into_tablespace_dir(self): + + corrupted_file = find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0] + + self.assertTrue( + corrupt_file(corrupted_file), + "ERROR: File is not corrupted or it missing" + ) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + +# # --- End ---# +# @unittest.skipUnless(ProbackupTest.enterprise, 'skip') +# def tearDown(self): +# self.node.cleanup() +# self.del_test_dir(module_name, self.fname) + + +#class CfsBackupEncTest(CfsBackupNoEncTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py new file mode 100644 index 00000000..73553a30 --- /dev/null +++ b/tests/cfs_restore.py @@ -0,0 +1,450 @@ +""" +restore + Syntax: + + pg_probackup restore -B backupdir --instance instance_name + [-D datadir] + [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] + [-j num_threads] [--progress] [-q] [-v] + +""" +import os +import unittest +import shutil + +from .helpers.cfs_helpers import find_by_name +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'cfs_restore' + +tblspace_name = 'cfs_tblspace' +tblspace_name_new = 'cfs_tblspace_new' + + +class CfsRestoreBase(ProbackupTest, unittest.TestCase): + def setUp(self): + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', +# 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.start() + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + self.add_data_in_cluster() + + self.backup_id = None + try: + self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + def add_data_in_cluster(self): + pass + + def tearDown(self): + self.node.cleanup() + self.del_test_dir(module_name, self.fname) + + +class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_empty_tablespace_from_fullbackup(self): + """ + Case: Restore empty tablespace from valid full backup. + """ + self.node.stop(["-m", "immediate"]) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), + "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." + ) + + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not restored or it restored without compressions" + ) + + +class CfsRestoreNoencTest(CfsRestoreBase): + def add_data_in_cluster(self): + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,1e5) i'.format('t1', tblspace_name) + ) + self.table_t1 = self.node.safe_psql( + "postgres", + "SELECT * FROM t1" + ) + + # --- Restore from full backup ---# + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in tablespace dir" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_3_jobs(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + self.node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + self.node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_5_jobs(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + self.node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id, options=['-j', '5']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + self.node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-j", "3", "-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_new_jobs(self): + pass + +# --------------------------------------------------------- # + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_new_jobs(self): + """ + Case: Restore from backup to new location, four jobs + """ + pass + + +#class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() +# +# +#class CfsRestoreEncTest(CfsRestoreNoencTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_validate_backup.py b/tests/cfs_validate_backup.py new file mode 100644 index 00000000..eea6f0e2 --- /dev/null +++ b/tests/cfs_validate_backup.py @@ -0,0 +1,25 @@ +import os +import unittest +import random + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_validate_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): + def setUp(self): + pass + + def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): + pass + + def tearDown(self): + pass + + +#class CfsValidateBackupNoenc(CfsValidateBackupNoenc): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsValidateBackupNoenc).setUp() diff --git a/tests/compression.py b/tests/compression.py new file mode 100644 index 00000000..aa275382 --- /dev/null +++ b/tests/compression.py @@ -0,0 +1,496 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'compression' + + +class CompressionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_compression_stream_zlib(self): + """make archive node, make full and page stream backups, check data correctness in restored instance""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=[ + '--stream', + '--compress-algorithm=zlib']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=[ + '--stream', '--compress-algorithm=zlib', + '--log-level-console=verbose', + '--log-level-file=verbose']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_archive_zlib(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=["--compress-algorithm=zlib"]) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--compress-algorithm=zlib"]) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_stream_pglz(self): + """ + make archive node, make full and page stream backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--compress-algorithm=pglz']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_archive_pglz(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--compress-algorithm=pglz']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_wrong_algorithm(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--compress-algorithm=bla-blah']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because compress-algorithm is invalid.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: invalid compress algorithm value "bla-blah"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delete_test.py b/tests/delete_test.py new file mode 100644 index 00000000..4afb15ae --- /dev/null +++ b/tests/delete_test.py @@ -0,0 +1,203 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from sys import exit + + +module_name = 'delete' + + +class DeleteTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_delete_full_backups(self): + """delete full backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + id_1 = show_backups[0]['id'] + id_2 = show_backups[1]['id'] + id_3 = show_backups[2]['id'] + self.delete_pb(backup_dir, 'node', id_2) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_3) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_increment_page(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_increment_ptrack(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_orphaned_wal_segments(self): + """make archive node, make three full backups, delete second backup without --wal option, then delete orphaned wals via --wal option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + # Check wals + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + original_wal_quantity = len(wals) + + # delete second full backup + self.delete_pb(backup_dir, 'node', backup_2_id) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + # try to delete wals for second backup + self.delete_pb(backup_dir, 'node', options=['--wal']) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # delete first full backup + self.delete_pb(backup_dir, 'node', backup_1_id) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + result = self.delete_pb(backup_dir, 'node', options=['--wal']) + # delete useless wals + self.assertTrue('INFO: removed min WAL segment' in result + and 'INFO: removed max WAL segment' in result) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # Check quantity, it should be lower than original + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + self.assertTrue(original_wal_quantity > len(wals), "Number of wals not changed after 'delete --wal' which is illegal") + + # Delete last backup + self.delete_pb(backup_dir, 'node', backup_3_id, options=['--wal']) + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + self.assertEqual (0, len(wals), "Number of wals should be equal to 0") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py new file mode 100644 index 00000000..40450016 --- /dev/null +++ b/tests/delta.py @@ -0,0 +1,1265 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import QueryException +import subprocess +import time + + +module_name = 'delta' + + +class DeltaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_1(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose" + ] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_2(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose", + "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_3(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10100000) i;" + ) + filepath = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')" + ).rstrip() + + self.backup_node(backup_dir, 'node', node) + + print(os.path.join(node.data_dir, filepath + '.1')) + os.unlink(os.path.join(node.data_dir, filepath + '.1')) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose" + ] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_stream(self): + """ + make archive node, take full and delta stream backups, + restore them and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_archive(self): + """ + make archive node, take full and delta archive backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check delta and data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init( + scale=100, + options=['--tablespace=somedata', '--no-vacuum']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # delta BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream']) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_full(self): + """ + make node, make full and delta stream backups, + restore them and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + # create async connection + conn = self.get_async_connect(port=node.port) + + self.wait(conn) + + acurs = conn.cursor() + acurs.execute("select pg_backend_pid()") + + self.wait(conn) + pid = acurs.fetchall()[0][0] + print(pid) + + gdb = self.gdb_attach(pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + if not gdb.continue_execution_until_running(): + print('Failed gdb continue') + exit(1) + + acurs.execute("VACUUM FULL t_heap") + + if gdb.stopped_in_breakpoint(): + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream'] + ) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take delta backup, + restore database and check it presense + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND DELTA BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exists_in_previous_backup(self): + """ + Make node, take full backup, create table, take page backup, + take delta backup, check that file is no fully copied to delta backup + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + filepath = node.safe_psql( + "postgres", + "SELECT pg_relation_filepath('t_heap')").rstrip() + self.backup_node( + backup_dir, + 'node', + node, + options=["--stream"]) + + # PAGE BACKUP + backup_id = self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + +# if self.paranoia: +# pgdata_page = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream", "--log-level-file=verbose"] + ) +# if self.paranoia: +# pgdata_delta = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) +# self.compare_pgdata( +# pgdata_page, pgdata_delta) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_delta(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new" + ) + + # DELTA BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ), + "--recovery-target-action=promote" + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_delta(self): + """ + Make node, take full backup, create database, + take delta backup, alter database tablespace location, + take delta backup restore last delta backup. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql( + "postgres", + "create database db1 tablespace = 'somedata'") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter database db1 set tablespace somedata_new" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_delete(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_1(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream", "--log-level-file=verbose"]) + + # open log file and check + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) + f.close + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', + "Backup Status should be OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_2(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of page " + "corruption in PostgreSQL instance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', + "Backup Status should be ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/exclude.py b/tests/exclude.py new file mode 100644 index 00000000..48b7889c --- /dev/null +++ b/tests/exclude.py @@ -0,0 +1,164 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'exclude' + + +class ExcludeTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_exclude_temp_tables(self): + """ + make node without archiving, create temp table, take full backup, + check that temp table not present in backup catalogue + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create temp table test as " + "select generate_series(0,50050000)::text") + conn.commit() + + temp_schema_name = conn.execute( + "SELECT nspname FROM pg_namespace " + "WHERE oid = pg_my_temp_schema()")[0][0] + conn.commit() + + temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( + "pg_", "") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + heap_path = conn.execute( + "select pg_relation_filepath('test')")[0][0] + conn.commit() + + index_path = conn.execute( + "select pg_relation_filepath('test_idx')")[0][0] + conn.commit() + + heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] + conn.commit() + + toast_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] + conn.commit() + + toast_idx_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, + "pg_toast_" + str(heap_oid) + "_index"))[0][0] + conn.commit() + + temp_table_filename = os.path.basename(heap_path) + temp_idx_filename = os.path.basename(index_path) + temp_toast_filename = os.path.basename(toast_path) + temp_idx_toast_filename = os.path.basename(toast_idx_path) + + self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file in [ + temp_table_filename, temp_table_filename + ".1", + temp_idx_filename, + temp_idx_filename + ".1", + temp_toast_filename, + temp_toast_filename + ".1", + temp_idx_toast_filename, + temp_idx_toast_filename + ".1" + ]: + self.assertEqual( + 1, 0, + "Found temp table file in backup catalogue.\n " + "Filepath: {0}".format(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exclude_unlogged_tables_1(self): + """ + make node without archiving, create unlogged table, take full backup, + alter table to unlogged, take ptrack backup, restore ptrack backup, + check that PGDATA`s are physically the same + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + "shared_buffers": "10MB", + "fsync": "off", + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create unlogged table test as " + "select generate_series(0,5005000)::text") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + node.safe_psql('postgres', "alter table test set logged") + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out new file mode 100644 index 00000000..35f58406 --- /dev/null +++ b/tests/expected/option_help.out @@ -0,0 +1,95 @@ + +pg_probackup - utility to manage backup/recovery of PostgreSQL database. + + pg_probackup help [COMMAND] + + pg_probackup version + + pg_probackup init -B backup-path + + pg_probackup set-config -B backup-dir --instance=instance_name + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup show-config -B backup-dir --instance=instance_name + [--format=format] + + pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + [-C] [--stream [-S slot-name]] [--backup-pg-log] + [-j num-threads] [--archive-timeout=archive-timeout] + [--progress] + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--delete-expired] [--delete-wal] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--compress] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup restore -B backup-dir --instance=instance_name + [-D pgdata-dir] [-i backup-id] [--progress] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--timeline=timeline] [-T OLDDIR=NEWDIR] + [--immediate] [--recovery-target-name=target-name] + [--recovery-target-action=pause|promote|shutdown] + [--restore-as-replica] + [--no-validate] + + pg_probackup validate -B backup-dir [--instance=instance_name] + [-i backup-id] [--progress] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--recovery-target-name=target-name] + [--timeline=timeline] + + pg_probackup show -B backup-dir + [--instance=instance_name [-i backup-id]] + [--format=format] + + pg_probackup delete -B backup-dir --instance=instance_name + [--wal] [-i backup-id | --expired] + + pg_probackup merge -B backup-dir --instance=instance_name + -i backup-id + + pg_probackup add-instance -B backup-dir -D pgdata-dir + --instance=instance_name + + pg_probackup del-instance -B backup-dir + --instance=instance_name + + pg_probackup archive-push -B backup-dir --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + [--compress [--compress-level=compress-level]] + [--overwrite] + + pg_probackup archive-get -B backup-dir --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + +Read the website for details. +Report bugs to . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out new file mode 100644 index 00000000..35e212c3 --- /dev/null +++ b/tests/expected/option_version.out @@ -0,0 +1 @@ +pg_probackup 2.0.18 \ No newline at end of file diff --git a/tests/false_positive.py b/tests/false_positive.py new file mode 100644 index 00000000..1884159b --- /dev/null +++ b/tests/false_positive.py @@ -0,0 +1,333 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + + +module_name = 'false_positive' + + +class FalsePositive(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_validate_wal_lost_segment(self): + """Loose segment located between backups. ExpectedFailure. This is BUG """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(backup_dir, "wal", 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = map(int, wals) + os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) + + # We just lost a wal segment and know nothing about it + self.backup_node(backup_dir, 'node', node) + self.assertTrue( + 'validation completed successfully' in self.validate_pb( + backup_dir, 'node')) + ######## + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.expectedFailure + # Need to force validation of ancestor-chain + def test_incremental_backup_corrupt_full_1(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", + backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + self.assertFalse( + True, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_ptrack_concurrent_get_and_clear_1(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('make_pagemap_from_ptrack') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + tablespace_oid = node.safe_psql( + "postgres", + "select oid from pg_tablespace where spcname = 'pg_default'").rstrip() + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap'::regclass::oid").rstrip() + + node.safe_psql( + "postgres", + "SELECT pg_ptrack_get_and_clear({0}, {1})".format( + tablespace_oid, relfilenode)) + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_ptrack_concurrent_get_and_clear_2(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('pthread_create') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + tablespace_oid = node.safe_psql( + "postgres", + "select oid from pg_tablespace " + "where spcname = 'pg_default'").rstrip() + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap'::regclass::oid").rstrip() + + node.safe_psql( + "postgres", + "SELECT pg_ptrack_get_and_clear({0}, {1})".format( + tablespace_oid, relfilenode)) + + gdb._execute("delete breakpoints") + gdb.continue_execution_until_exit() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup ptrack_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_multiple_delete(self): + """delete multiple backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + self.delete_pb(backup_dir, 'node', options= + ["-i {0}".format(backup_1_id), "-i {0}".format(backup_2_id), "-i {0}".format(backup_3_id)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..ac64c423 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,2 @@ +__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] +#from . import * \ No newline at end of file diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py new file mode 100644 index 00000000..67e2b331 --- /dev/null +++ b/tests/helpers/cfs_helpers.py @@ -0,0 +1,91 @@ +import os +import re +import random +import string + + +def find_by_extensions(dirs=None, extensions=None): + """ + find_by_extensions(['path1','path2'],['.txt','.log']) + :return: + Return list of files include full path by file extensions + """ + files = [] + new_dirs = [] + + if dirs is not None and extensions is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.splitext(d)[1] in extensions: + files.append(d) + + if new_dirs: + files.extend(find_by_extensions(new_dirs, extensions)) + + return files + + +def find_by_pattern(dirs=None, pattern=None): + """ + find_by_pattern(['path1','path2'],'^.*/*.txt') + :return: + Return list of files include full path by pattern + """ + files = [] + new_dirs = [] + + if dirs is not None and pattern is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if re.match(pattern,d): + files.append(d) + + if new_dirs: + files.extend(find_by_pattern(new_dirs, pattern)) + + return files + + +def find_by_name(dirs=None, filename=None): + files = [] + new_dirs = [] + + if dirs is not None and filename is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.basename(d) in filename: + files.append(d) + + if new_dirs: + files.extend(find_by_name(new_dirs, filename)) + + return files + + +def corrupt_file(filename): + file_size = None + try: + file_size = os.path.getsize(filename) + except OSError: + return False + + try: + with open(filename, "rb+") as f: + f.seek(random.randint(int(0.1*file_size),int(0.8*file_size))) + f.write(random_string(0.1*file_size)) + f.close() + except OSError: + return False + + return True + + +def random_string(n): + a = string.ascii_letters + string.digits + return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py new file mode 100644 index 00000000..0d04d898 --- /dev/null +++ b/tests/helpers/ptrack_helpers.py @@ -0,0 +1,1300 @@ +# you need os for unittest to work +import os +from sys import exit, argv, version_info +import subprocess +import shutil +import six +import testgres +import hashlib +import re +import pwd +import select +import psycopg2 +from time import sleep +import re +import json + +idx_ptrack = { + 't_heap': { + 'type': 'heap' + }, + 't_btree': { + 'type': 'btree', + 'column': 'text', + 'relation': 't_heap' + }, + 't_seq': { + 'type': 'seq', + 'column': 't_seq', + 'relation': 't_heap' + }, + 't_spgist': { + 'type': 'spgist', + 'column': 'text', + 'relation': 't_heap' + }, + 't_brin': { + 'type': 'brin', + 'column': 'text', + 'relation': 't_heap' + }, + 't_gist': { + 'type': 'gist', + 'column': 'tsvector', + 'relation': 't_heap' + }, + 't_gin': { + 'type': 'gin', + 'column': 'tsvector', + 'relation': 't_heap' + }, +} + +archive_script = """ +#!/bin/bash +count=$(ls {backup_dir}/test00* | wc -l) +if [ $count -ge {count_limit} ] +then + exit 1 +else + cp $1 {backup_dir}/wal/{node_name}/$2 + count=$((count+1)) + touch {backup_dir}/test00$count + exit 0 +fi +""" +warning = """ +Wrong splint in show_pb +Original Header: +{header} +Original Body: +{body} +Splitted Header +{header_split} +Splitted Body +{body_split} +""" + + +def dir_files(base_dir): + out_list = [] + for dir_name, subdir_list, file_list in os.walk(base_dir): + if dir_name != base_dir: + out_list.append(os.path.relpath(dir_name, base_dir)) + for fname in file_list: + out_list.append( + os.path.relpath(os.path.join( + dir_name, fname), base_dir) + ) + out_list.sort() + return out_list + + +def is_enterprise(): + # pg_config --help + p = subprocess.Popen( + [os.environ['PG_CONFIG'], '--help'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if b'postgrespro.ru' in p.communicate()[0]: + return True + else: + return False + + +class ProbackupException(Exception): + def __init__(self, message, cmd): + self.message = message + self.cmd = cmd + + def __str__(self): + return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) + + +def slow_start(self, replica=False): + + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # raise_operational_error=False) + + self.start() + if not replica: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + else: + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + +# while True: +# try: +# self.poll_query_until( +# "postgres", +# "SELECT pg_is_in_recovery()") +# break +# except ProbackupException as e: +# continue + + +class ProbackupTest(object): + # Class attributes + enterprise = is_enterprise() + + def __init__(self, *args, **kwargs): + super(ProbackupTest, self).__init__(*args, **kwargs) + if '-v' in argv or '--verbose' in argv: + self.verbose = True + else: + self.verbose = False + + self.test_env = os.environ.copy() + envs_list = [ + "LANGUAGE", + "LC_ALL", + "PGCONNECT_TIMEOUT", + "PGDATA", + "PGDATABASE", + "PGHOSTADDR", + "PGREQUIRESSL", + "PGSERVICE", + "PGSSLMODE", + "PGUSER", + "PGPORT", + "PGHOST" + ] + + for e in envs_list: + try: + del self.test_env[e] + except: + pass + + self.test_env["LC_MESSAGES"] = "C" + self.test_env["LC_TIME"] = "C" + + self.paranoia = False + if 'PG_PROBACKUP_PARANOIA' in self.test_env: + if self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON': + self.paranoia = True + + self.archive_compress = False + if 'ARCHIVE_COMPRESSION' in self.test_env: + if self.test_env['ARCHIVE_COMPRESSION'] == 'ON': + self.archive_compress = True + try: + testgres.configure_testgres( + cache_initdb=False, + cached_initdb_dir=False, + cache_pg_config=False, + node_cleanup_full=False) + except: + pass + + self.helpers_path = os.path.dirname(os.path.realpath(__file__)) + self.dir_path = os.path.abspath( + os.path.join(self.helpers_path, os.pardir) + ) + self.tmp_path = os.path.abspath( + os.path.join(self.dir_path, 'tmp_dirs') + ) + try: + os.makedirs(os.path.join(self.dir_path, 'tmp_dirs')) + except: + pass + + self.user = self.get_username() + self.probackup_path = None + if "PGPROBACKUPBIN" in self.test_env: + if ( + os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and + os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) + ): + self.probackup_path = self.test_env["PGPROBACKUPBIN"] + else: + if self.verbose: + print('PGPROBINDIR is not an executable file') + if not self.probackup_path: + self.probackup_path = os.path.abspath(os.path.join( + self.dir_path, "../pg_probackup")) + + def make_simple_node( + self, + base_dir=None, + set_replication=False, + initdb_params=[], + pg_options={}): + + real_base_dir = os.path.join(self.tmp_path, base_dir) + shutil.rmtree(real_base_dir, ignore_errors=True) + os.makedirs(real_base_dir) + + node = testgres.get_new_node('test', base_dir=real_base_dir) + # bound method slow_start() to 'node' class instance + node.slow_start = slow_start.__get__(node) + node.should_rm_dirs = True + node.init( + initdb_params=initdb_params, allow_streaming=set_replication) + + # Sane default parameters + node.append_conf("postgresql.auto.conf", "max_connections = 100") + node.append_conf("postgresql.auto.conf", "shared_buffers = 10MB") + node.append_conf("postgresql.auto.conf", "fsync = on") + node.append_conf("postgresql.auto.conf", "wal_level = logical") + node.append_conf("postgresql.auto.conf", "hot_standby = 'off'") + + node.append_conf( + "postgresql.auto.conf", "log_line_prefix = '%t [%p]: [%l-1] '") + node.append_conf("postgresql.auto.conf", "log_statement = none") + node.append_conf("postgresql.auto.conf", "log_duration = on") + node.append_conf( + "postgresql.auto.conf", "log_min_duration_statement = 0") + node.append_conf("postgresql.auto.conf", "log_connections = on") + node.append_conf("postgresql.auto.conf", "log_disconnections = on") + + # Apply given parameters + for key, value in six.iteritems(pg_options): + node.append_conf("postgresql.auto.conf", "%s = %s" % (key, value)) + + # Allow replication in pg_hba.conf + if set_replication: + node.append_conf( + "pg_hba.conf", + "local replication all trust\n") + node.append_conf( + "postgresql.auto.conf", + "max_wal_senders = 10") + + return node + + def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): + res = node.execute( + "postgres", + "select exists" + " (select 1 from pg_tablespace where spcname = '{0}')".format( + tblspc_name) + ) + # Check that tablespace with name 'tblspc_name' do not exists already + self.assertFalse( + res[0][0], + 'Tablespace "{0}" already exists'.format(tblspc_name) + ) + + if not tblspc_path: + tblspc_path = os.path.join( + node.base_dir, '{0}'.format(tblspc_name)) + cmd = "CREATE TABLESPACE {0} LOCATION '{1}'".format( + tblspc_name, tblspc_path) + if cfs: + cmd += " with (compression=true)" + + if not os.path.exists(tblspc_path): + os.makedirs(tblspc_path) + res = node.safe_psql("postgres", cmd) + # Check that tablespace was successfully created + # self.assertEqual( + # res[0], 0, + # 'Failed to create tablespace with cmd: {0}'.format(cmd)) + + def get_tblspace_path(self, node, tblspc_name): + return os.path.join(node.base_dir, tblspc_name) + + def get_fork_size(self, node, fork_name): + return node.execute( + "postgres", + "select pg_relation_size('{0}')/8192".format(fork_name))[0][0] + + def get_fork_path(self, node, fork_name): + return os.path.join( + node.base_dir, 'data', node.execute( + "postgres", + "select pg_relation_filepath('{0}')".format( + fork_name))[0][0] + ) + + def get_md5_per_page_for_fork(self, file, size_in_pages): + pages_per_segment = {} + md5_per_page = {} + nsegments = size_in_pages/131072 + if size_in_pages % 131072 != 0: + nsegments = nsegments + 1 + + size = size_in_pages + for segment_number in range(nsegments): + if size - 131072 > 0: + pages_per_segment[segment_number] = 131072 + else: + pages_per_segment[segment_number] = size + size = size - 131072 + + for segment_number in range(nsegments): + offset = 0 + if segment_number == 0: + file_desc = os.open(file, os.O_RDONLY) + start_page = 0 + end_page = pages_per_segment[segment_number] + else: + file_desc = os.open( + file+".{0}".format(segment_number), os.O_RDONLY + ) + start_page = max(md5_per_page)+1 + end_page = end_page + pages_per_segment[segment_number] + + for page in range(start_page, end_page): + md5_per_page[page] = hashlib.md5( + os.read(file_desc, 8192)).hexdigest() + offset += 8192 + os.lseek(file_desc, offset, 0) + os.close(file_desc) + + return md5_per_page + + def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): + + if self.get_pgpro_edition(node) == 'enterprise': + header_size = 48 + else: + header_size = 24 + ptrack_bits_for_fork = [] + + page_body_size = 8192-header_size + byte_size = os.path.getsize(file + '_ptrack') + npages = byte_size/8192 + if byte_size % 8192 != 0: + print('Ptrack page is not 8k aligned') + sys.exit(1) + + file = os.open(file + '_ptrack', os.O_RDONLY) + + for page in range(npages): + offset = 8192*page+header_size + os.lseek(file, offset, 0) + lots_of_bytes = os.read(file, page_body_size) + byte_list = [ + lots_of_bytes[i:i+1] for i in range(len(lots_of_bytes)) + ] + for byte in byte_list: + # byte_inverted = bin(int(byte, base=16))[2:][::-1] + # bits = (byte >> x) & 1 for x in range(7, -1, -1) + byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1] + for bit in byte_inverted: + # if len(ptrack_bits_for_fork) < size: + ptrack_bits_for_fork.append(int(bit)) + + os.close(file) + return ptrack_bits_for_fork + + def check_ptrack_sanity(self, idx_dict): + success = True + if idx_dict['new_size'] > idx_dict['old_size']: + size = idx_dict['new_size'] + else: + size = idx_dict['old_size'] + for PageNum in range(size): + if PageNum not in idx_dict['old_pages']: + # Page was not present before, meaning that relation got bigger + # Ptrack should be equal to 1 + if idx_dict['ptrack'][PageNum] != 1: + if self.verbose: + print( + 'Page Number {0} of type {1} was added,' + ' but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + # print(idx_dict) + success = False + continue + if PageNum not in idx_dict['new_pages']: + # Page is not present now, meaning that relation got smaller + # Ptrack should be equal to 0, + # We are not freaking out about false positive stuff + if idx_dict['ptrack'][PageNum] != 0: + if self.verbose: + print( + 'Page Number {0} of type {1} was deleted,' + ' but ptrack value is {2}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + continue + + # Ok, all pages in new_pages that do not have + # corresponding page in old_pages are been dealt with. + # We can now safely proceed to comparing old and new pages + if idx_dict['new_pages'][ + PageNum] != idx_dict['old_pages'][PageNum]: + # Page has been changed, + # meaning that ptrack should be equal to 1 + if idx_dict['ptrack'][PageNum] != 1: + if self.verbose: + print( + 'Page Number {0} of type {1} was changed,' + ' but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + print( + "\n Old checksumm: {0}\n" + " New checksumm: {1}".format( + idx_dict['old_pages'][PageNum], + idx_dict['new_pages'][PageNum]) + ) + + if PageNum == 0 and idx_dict['type'] == 'spgist': + if self.verbose: + print( + 'SPGIST is a special snowflake, so don`t ' + 'fret about losing ptrack for blknum 0' + ) + continue + success = False + else: + # Page has not been changed, + # meaning that ptrack should be equal to 0 + if idx_dict['ptrack'][PageNum] != 0: + if self.verbose: + print( + 'Page Number {0} of type {1} was not changed,' + ' but ptrack value is {2}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum] + ) + ) + + self.assertTrue( + success, 'Ptrack does not correspond to state' + ' of its own pages.\n Gory Details: \n{0}'.format( + idx_dict['type'], idx_dict + ) + ) + + def check_ptrack_recovery(self, idx_dict): + size = idx_dict['size'] + for PageNum in range(size): + if idx_dict['ptrack'][PageNum] != 1: + self.assertTrue( + False, + 'Recovery for Page Number {0} of Type {1}' + ' was conducted, but ptrack value is {2}.' + ' THIS IS BAD\n IDX_DICT: {3}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum], + idx_dict + ) + ) + + def check_ptrack_clean(self, idx_dict, size): + for PageNum in range(size): + if idx_dict['ptrack'][PageNum] != 0: + self.assertTrue( + False, + 'Ptrack for Page Number {0} of Type {1}' + ' should be clean, but ptrack value is {2}.' + '\n THIS IS BAD\n IDX_DICT: {3}'.format( + PageNum, + idx_dict['type'], + idx_dict['ptrack'][PageNum], + idx_dict + ) + ) + + def run_pb(self, command, async=False, gdb=False): + try: + self.cmd = [' '.join(map(str, [self.probackup_path] + command))] + if self.verbose: + print(self.cmd) + if gdb: + return GDBobj([self.probackup_path] + command, self.verbose) + if async: + return subprocess.Popen( + self.cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + self.output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") + if command[0] == 'backup': + # return backup ID + for line in self.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + else: + return self.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode("utf-8"), self.cmd) + + def run_binary(self, command, async=False): + if self.verbose: + print([' '.join(map(str, command))]) + try: + if async: + return subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + self.output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") + return self.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode("utf-8"), command) + + def init_pb(self, backup_dir): + + shutil.rmtree(backup_dir, ignore_errors=True) + return self.run_pb([ + "init", + "-B", backup_dir + ]) + + def add_instance(self, backup_dir, instance, node): + + return self.run_pb([ + "add-instance", + "--instance={0}".format(instance), + "-B", backup_dir, + "-D", node.data_dir + ]) + + def del_instance(self, backup_dir, instance): + + return self.run_pb([ + "del-instance", + "--instance={0}".format(instance), + "-B", backup_dir + ]) + + def clean_pb(self, backup_dir): + shutil.rmtree(backup_dir, ignore_errors=True) + + def backup_node( + self, backup_dir, instance, node, data_dir=False, + backup_type="full", options=[], async=False, gdb=False + ): + if not node and not data_dir: + print('You must provide ether node or data_dir for backup') + exit(1) + + if node: + pgdata = node.data_dir + + if data_dir: + pgdata = data_dir + + cmd_list = [ + "backup", + "-B", backup_dir, + # "-D", pgdata, + "-p", "%i" % node.port, + "-d", "postgres", + "--instance={0}".format(instance) + ] + if backup_type: + cmd_list += ["-b", backup_type] + + return self.run_pb(cmd_list + options, async, gdb) + + def merge_backup(self, backup_dir, instance, backup_id): + cmd_list = [ + "merge", + "-B", backup_dir, + "--instance={0}".format(instance), + "-i", backup_id + ] + + return self.run_pb(cmd_list) + + def restore_node( + self, backup_dir, instance, node=False, + data_dir=None, backup_id=None, options=[] + ): + if data_dir is None: + data_dir = node.data_dir + + cmd_list = [ + "restore", + "-B", backup_dir, + "-D", data_dir, + "--instance={0}".format(instance) + ] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def show_pb( + self, backup_dir, instance=None, backup_id=None, + options=[], as_text=False, as_json=True + ): + + backup_list = [] + specific_record = {} + cmd_list = [ + "show", + "-B", backup_dir, + ] + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if backup_id: + cmd_list += ["-i", backup_id] + + if as_json: + cmd_list += ["--format=json"] + + if as_text: + # You should print it when calling as_text=true + return self.run_pb(cmd_list + options) + + # get show result as list of lines + if as_json: + data = json.loads(self.run_pb(cmd_list + options)) + # print(data) + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: + continue + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) + return backup_list + else: + show_splitted = self.run_pb(cmd_list + options).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(" +", header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(" +", backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(" = ")[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + return specific_record + + def validate_pb( + self, backup_dir, instance=None, + backup_id=None, options=[] + ): + + cmd_list = [ + "validate", + "-B", backup_dir + ] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def delete_pb(self, backup_dir, instance, backup_id=None, options=[]): + cmd_list = [ + "delete", + "-B", backup_dir + ] + + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def delete_expired(self, backup_dir, instance, options=[]): + cmd_list = [ + "delete", "--expired", "--wal", + "-B", backup_dir, + "--instance={0}".format(instance) + ] + return self.run_pb(cmd_list + options) + + def show_config(self, backup_dir, instance): + out_dict = {} + cmd_list = [ + "show-config", + "-B", backup_dir, + "--instance={0}".format(instance) + ] + res = self.run_pb(cmd_list).splitlines() + for line in res: + if not line.startswith('#'): + name, var = line.partition(" = ")[::2] + out_dict[name] = var + return out_dict + + def get_recovery_conf(self, node): + out_dict = {} + with open( + os.path.join(node.data_dir, "recovery.conf"), "r" + ) as recovery_conf: + for line in recovery_conf: + try: + key, value = line.split("=") + except: + continue + out_dict[key.strip()] = value.strip(" '").replace("'\n", "") + return out_dict + + def set_archiving( + self, backup_dir, instance, node, replica=False, overwrite=False): + + if replica: + archive_mode = 'always' + node.append_conf('postgresql.auto.conf', 'hot_standby = on') + else: + archive_mode = 'on' + + # node.append_conf( + # "postgresql.auto.conf", + # "wal_level = archive" + # ) + node.append_conf( + "postgresql.auto.conf", + "archive_mode = {0}".format(archive_mode) + ) + archive_command = "{0} archive-push -B {1} --instance={2} ".format( + self.probackup_path, backup_dir, instance) + + if os.name == 'posix': + if self.archive_compress: + archive_command = archive_command + "--compress " + + if overwrite: + archive_command = archive_command + "--overwrite " + + archive_command = archive_command + "--wal-file-path %p --wal-file-name %f" + + node.append_conf( + "postgresql.auto.conf", + "archive_command = '{0}'".format( + archive_command)) + # elif os.name == 'nt': + # node.append_conf( + # "postgresql.auto.conf", + # "archive_command = 'copy %p {0}\\%f'".format(archive_dir) + # ) + + def set_replica( + self, master, replica, + replica_name='replica', + synchronous=False + ): + replica.append_conf( + "postgresql.auto.conf", "port = {0}".format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf( + "recovery.conf", + "primary_conninfo = 'user={0} port={1} application_name={2}" + " sslmode=prefer sslcompression=1'".format( + self.user, master.port, replica_name) + ) + if synchronous: + master.append_conf( + "postgresql.auto.conf", + "synchronous_standby_names='{0}'".format(replica_name) + ) + master.append_conf( + 'postgresql.auto.conf', + "synchronous_commit='remote_apply'" + ) + master.reload() + + def wrong_wal_clean(self, node, wal_size): + wals_dir = os.path.join(self.backup_dir(node), "wal") + wals = [ + f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) + ] + wals.sort() + file_path = os.path.join(wals_dir, wals[-1]) + if os.path.getsize(file_path) != wal_size: + os.remove(file_path) + + def guc_wal_segment_size(self, node): + var = node.execute( + "postgres", + "select setting from pg_settings where name = 'wal_segment_size'" + ) + return int(var[0][0]) * self.guc_wal_block_size(node) + + def guc_wal_block_size(self, node): + var = node.execute( + "postgres", + "select setting from pg_settings where name = 'wal_block_size'" + ) + return int(var[0][0]) + + def get_pgpro_edition(self, node): + if node.execute( + "postgres", + "select exists (select 1 from" + " pg_proc where proname = 'pgpro_edition')" + )[0][0]: + var = node.execute("postgres", "select pgpro_edition()") + return str(var[0][0]) + else: + return False + + def get_username(self): + """ Returns current user name """ + return pwd.getpwuid(os.getuid())[0] + + def version_to_num(self, version): + if not version: + return 0 + parts = version.split(".") + while len(parts) < 3: + parts.append("0") + num = 0 + for part in parts: + num = num * 100 + int(re.sub("[^\d]", "", part)) + return num + + def switch_wal_segment(self, node): + """ + Execute pg_switch_wal/xlog() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + if self.version_to_num( + node.safe_psql("postgres", "show server_version") + ) >= self.version_to_num('10.0'): + node.safe_psql("postgres", "select pg_switch_wal()") + else: + node.safe_psql("postgres", "select pg_switch_xlog()") + else: + if self.version_to_num( + node.execute("show server_version")[0][0] + ) >= self.version_to_num('10.0'): + node.execute("select pg_switch_wal()") + else: + node.execute("select pg_switch_xlog()") + sleep(1) + + def get_version(self, node): + return self.version_to_num( + testgres.get_pg_config()["VERSION"].split(" ")[1]) + + def get_bin_path(self, binary): + return testgres.get_bin_path(binary) + + def del_test_dir(self, module_name, fname): + """ Del testdir and optimistically try to del module dir""" + try: + testgres.clean_all() + except: + pass + + shutil.rmtree( + os.path.join( + self.tmp_path, + module_name, + fname + ), + ignore_errors=True + ) + try: + os.rmdir(os.path.join(self.tmp_path, module_name)) + except: + pass + + def pgdata_content(self, directory, ignore_ptrack=True): + """ return dict with directory content. " + " TAKE IT AFTER CHECKPOINT or BACKUP""" + dirs_to_ignore = [ + 'pg_xlog', 'pg_wal', 'pg_log', + 'pg_stat_tmp', 'pg_subtrans', 'pg_notify' + ] + files_to_ignore = [ + 'postmaster.pid', 'postmaster.opts', + 'pg_internal.init', 'postgresql.auto.conf', + 'backup_label', 'tablespace_map', 'recovery.conf', + 'ptrack_control', 'ptrack_init', 'pg_control' + ] +# suffixes_to_ignore = ( +# '_ptrack' +# ) + directory_dict = {} + directory_dict['pgdata'] = directory + directory_dict['files'] = {} + for root, dirs, files in os.walk(directory, followlinks=True): + dirs[:] = [d for d in dirs if d not in dirs_to_ignore] + for file in files: + if ( + file in files_to_ignore or + (ignore_ptrack and file.endswith('_ptrack')) + ): + continue + + file_fullpath = os.path.join(root, file) + file_relpath = os.path.relpath(file_fullpath, directory) + directory_dict['files'][file_relpath] = {'is_datafile': False} + directory_dict['files'][file_relpath]['md5'] = hashlib.md5( + open(file_fullpath, 'rb').read()).hexdigest() + + if file.isdigit(): + directory_dict['files'][file_relpath]['is_datafile'] = True + size_in_pages = os.path.getsize(file_fullpath)/8192 + directory_dict['files'][file_relpath][ + 'md5_per_page'] = self.get_md5_per_page_for_fork( + file_fullpath, size_in_pages + ) + + return directory_dict + + def compare_pgdata(self, original_pgdata, restored_pgdata): + """ return dict with directory content. DO IT BEFORE RECOVERY""" + fail = False + error_message = 'Restored PGDATA is not equal to original!\n' + for file in restored_pgdata['files']: + # File is present in RESTORED PGDATA + # but not present in ORIGINAL + # only backup_label is allowed + if file not in original_pgdata['files']: + fail = True + error_message += '\nFile is not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], file)) + + for file in original_pgdata['files']: + if file in restored_pgdata['files']: + + if ( + original_pgdata['files'][file]['md5'] != + restored_pgdata['files'][file]['md5'] + ): + fail = True + error_message += ( + '\nFile Checksumm mismatch.\n' + 'File_old: {0}\nChecksumm_old: {1}\n' + 'File_new: {2}\nChecksumm_new: {3}\n').format( + os.path.join(original_pgdata['pgdata'], file), + original_pgdata['files'][file]['md5'], + os.path.join(restored_pgdata['pgdata'], file), + restored_pgdata['files'][file]['md5'] + ) + + if original_pgdata['files'][file]['is_datafile']: + for page in original_pgdata['files'][file]['md5_per_page']: + if page not in restored_pgdata['files'][file]['md5_per_page']: + error_message += ( + '\n Page {0} dissappeared.\n ' + 'File: {1}\n').format( + page, + os.path.join( + restored_pgdata['pgdata'], + file + ) + ) + continue + + if original_pgdata['files'][file][ + 'md5_per_page'][page] != restored_pgdata[ + 'files'][file]['md5_per_page'][page]: + error_message += ( + '\n Page checksumm mismatch: {0}\n ' + ' PAGE Checksumm_old: {1}\n ' + ' PAGE Checksumm_new: {2}\n ' + ' File: {3}\n' + ).format( + page, + original_pgdata['files'][file][ + 'md5_per_page'][page], + restored_pgdata['files'][file][ + 'md5_per_page'][page], + os.path.join( + restored_pgdata['pgdata'], file) + ) + for page in restored_pgdata['files'][file]['md5_per_page']: + if page not in original_pgdata['files'][file]['md5_per_page']: + error_message += '\n Extra page {0}\n File: {1}\n'.format( + page, + os.path.join( + restored_pgdata['pgdata'], file)) + + else: + error_message += ( + '\nFile dissappearance.\n ' + 'File: {0}\n').format( + os.path.join(restored_pgdata['pgdata'], file) + ) + fail = True + self.assertFalse(fail, error_message) + + def get_async_connect(self, database=None, host=None, port=5432): + if not database: + database = 'postgres' + if not host: + host = '127.0.0.1' + + return psycopg2.connect( + database="postgres", + host='127.0.0.1', + port=port, + async=True + ) + + def wait(self, connection): + while True: + state = connection.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [connection.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([connection.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + + def gdb_attach(self, pid): + return GDBobj([str(pid)], self.verbose, attach=True) + + +class GdbException(Exception): + def __init__(self, message=False): + self.message = message + + def __str__(self): + return '\n ERROR: {0}\n'.format(repr(self.message)) + + +class GDBobj(ProbackupTest): + def __init__(self, cmd, verbose, attach=False): + self.verbose = verbose + + # Check gdb presense + try: + gdb_version, _ = subprocess.Popen( + ["gdb", "--version"], + stdout=subprocess.PIPE + ).communicate() + except OSError: + raise GdbException("Couldn't find gdb on the path") + + self.base_cmd = [ + 'gdb', + '--interpreter', + 'mi2', + ] + + if attach: + self.cmd = self.base_cmd + ['--pid'] + cmd + else: + self.cmd = self.base_cmd + ['--args'] + cmd + + # Get version + gdb_version_number = re.search( + b"^GNU gdb [^\d]*(\d+)\.(\d)", + gdb_version) + self.major_version = int(gdb_version_number.group(1)) + self.minor_version = int(gdb_version_number.group(2)) + + if self.verbose: + print([' '.join(map(str, self.cmd))]) + + self.proc = subprocess.Popen( + self.cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + universal_newlines=True + ) + self.gdb_pid = self.proc.pid + + # discard data from pipe, + # is there a way to do it a less derpy way? + while True: + line = self.proc.stdout.readline() + + if 'No such process' in line: + raise GdbException(line) + + if not line.startswith('(gdb)'): + pass + else: + break + + def set_breakpoint(self, location): + result = self._execute('break ' + location) + for line in result: + if line.startswith('~"Breakpoint'): + return + + elif line.startswith('^error') or line.startswith('(gdb)'): + break + + elif line.startswith('&"break'): + pass + + elif line.startswith('&"Function'): + raise GdbException(line) + + elif line.startswith('&"No line'): + raise GdbException(line) + + elif line.startswith('~"Make breakpoint pending on future shared'): + raise GdbException(line) + + raise GdbException( + 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + ) + + def run_until_break(self): + result = self._execute('run', False) + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + raise GdbException( + 'Failed to run until breakpoint.\n' + ) + + def continue_execution_until_running(self): + result = self._execute('continue') + + running = False + for line in result: + if line.startswith('*running'): + running = True + break + if line.startswith('*stopped,reason="breakpoint-hit"'): + running = False + continue + if line.startswith('*stopped,reason="exited-normally"'): + running = False + continue + return running + + def continue_execution_until_exit(self): + result = self._execute('continue', False) + + for line in result: + if line.startswith('*running'): + continue + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if ( + line.startswith('*stopped,reason="exited-normally"') or + line == '*stopped\n' + ): + return + raise GdbException( + 'Failed to continue execution until exit.\n' + ) + + def continue_execution_until_break(self, ignore_count=0): + if ignore_count > 0: + result = self._execute( + 'continue ' + str(ignore_count), + False + ) + else: + result = self._execute('continue', False) + + running = False + for line in result: + if line.startswith('*running'): + running = True + if line.startswith('*stopped,reason="breakpoint-hit"'): + return 'breakpoint-hit' + if line.startswith('*stopped,reason="exited-normally"'): + return 'exited-normally' + if running: + return 'running' + + def stopped_in_breakpoint(self): + output = [] + while True: + line = self.proc.stdout.readline() + output += [line] + if self.verbose: + print(line) + if line.startswith('*stopped,reason="breakpoint-hit"'): + return True + return False + + # use for breakpoint, run, continue + def _execute(self, cmd, running=True): + output = [] + self.proc.stdin.flush() + self.proc.stdin.write(cmd + '\n') + self.proc.stdin.flush() + + while True: + line = self.proc.stdout.readline() + output += [line] + if self.verbose: + print(repr(line)) + if line == '^done\n' or line.startswith('*stopped'): + break + if running and line.startswith('*running'): + break + return output diff --git a/tests/init_test.py b/tests/init_test.py new file mode 100644 index 00000000..0b91dafa --- /dev/null +++ b/tests/init_test.py @@ -0,0 +1,99 @@ +import os +import unittest +from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException + + +module_name = 'init' + + +class InitTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_success(self): + """Success normal init""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + self.init_pb(backup_dir) + self.assertEqual( + dir_files(backup_dir), + ['backups', 'wal'] + ) + self.add_instance(backup_dir, 'node', node) + self.assertEqual("INFO: Instance 'node' successfully deleted\n", self.del_instance(backup_dir, 'node'), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Show non-existing instance + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Delete non-existing instance + try: + self.del_instance(backup_dir, 'node1') + self.assertEqual(1, 0, 'Expecting Error due to delete of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node1' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Add instance without pgdata + try: + self.run_pb([ + "add-instance", + "--instance=node1", + "-B", backup_dir + ]) + self.assertEqual(1, 0, 'Expecting Error due to adding instance without pgdata. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_already_exist(self): + """Failure with backup catalog already existed""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + self.init_pb(backup_dir) + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_abs_path(self): + """failure with backup catalog should be given as absolute path""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + try: + self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) + self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: -B, --backup-path must be an absolute path\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/logging.py b/tests/logging.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/merge.py b/tests/merge.py new file mode 100644 index 00000000..1be3dd8b --- /dev/null +++ b/tests/merge.py @@ -0,0 +1,454 @@ +# coding: utf-8 + +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest + +module_name = "merge" + + +class MergeTest(ProbackupTest, unittest.TestCase): + + def test_merge_full_page(self): + """ + Test MERGE command, it merges FULL backup with target PAGE backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + conn.commit() + + # Do first page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do second page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + # sanity check + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Check physical correctness + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_backups(self): + """ + Test MERGE command with compressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full compressed backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do compressed page backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[1] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_merge_tablespaces(self): + """ + Some test here + """ + + def test_merge_page_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_delta_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_ptrack_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/option_test.py b/tests/option_test.py new file mode 100644 index 00000000..8bd473fa --- /dev/null +++ b/tests/option_test.py @@ -0,0 +1,218 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'option' + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_help_1(self): + """help options""" + self.maxDiff = None + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + + # @unittest.skip("skip") + def test_version_2(self): + """help options""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: + self.assertIn( + version_out.read().decode("utf-8"), + self.run_pb(["--version"]) + ) + + # @unittest.skip("skip") + def test_without_backup_path_3(self): + """backup command failure without backup mode option""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + try: + self.run_pb(["backup", "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # @unittest.skip("skip") + def test_options_4(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # backup command failure without instance option + try: + self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: required parameter not specified: --instance\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure without backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) + self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn('ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure with invalid backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) + self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid backup-mode "bad"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # delete failure without delete options + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because delete options are omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: You must specify at least one of the delete options: --expired |--wal |--backup_id\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # delete failure without ID + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node", '-i']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue("option requires an argument -- 'i'" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_options_5(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + pg_options={ + 'wal_level': 'logical', + 'max_wal_senders': '2'}) + + self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir), + self.init_pb(backup_dir)) + self.add_instance(backup_dir, 'node', node) + + node.start() + + # syntax error in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write(" = INFINITE\n") + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: syntax error in " = INFINITE"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid value in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("BACKUP_MODE=\n") + + try: + self.backup_node(backup_dir, 'node', node, backup_type=None), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid backup-mode ""\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Command line parameters should override file values + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy=1\n") + + self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') + + # User cannot send --system-identifier parameter via command line + try: + self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: option system-identifier cannot be specified in command line\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # invalid value in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("SMOOTH_CHECKPOINT=FOO\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid option in pg_probackup.conf + pbconf_path = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") + with open(pbconf_path, "a") as conf: + conf.write("TIMELINEID=1\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid option "TIMELINEID" in file "{0}"\n'.format(pbconf_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/page.py b/tests/page.py new file mode 100644 index 00000000..ef7122b6 --- /dev/null +++ b/tests/page.py @@ -0,0 +1,641 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + +module_name = 'page' + + +class PageBackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_page_vacuum_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, take second page backup, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--log-level-file=verbose']) + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_stream(self): + """ + make archive node, take full and page stream backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_archive(self): + """ + make archive node, take full and page archive backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full') + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn("INFO: Restore of backup {0} completed.".format( + full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check page and data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--log-level-file=verbose"]) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", + "--recovery-target-action=promote", + "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_delete(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + # FULL backup + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_delete_1(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + "hot_standby": "on" + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + with node.connect() as conn: + conn.execute("create table test (id int)") + for x in range(0, 8): + conn.execute( + "insert into test select i from generate_series(1,100) s(i)") + conn.commit() + self.switch_wal_segment(conn) + count1 = conn.execute("select count(*) from test") + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore it + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Check restored node + count2 = node_restored.execute("postgres", "select count(*) from test") + + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + node_restored.cleanup() + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap_1(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + node.pgbench_init(scale=10) + + # do page backup in single thread + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + self.delete_pb(backup_dir, 'node', page_id) + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.start() + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro560.py b/tests/pgpro560.py new file mode 100644 index 00000000..bf334556 --- /dev/null +++ b/tests/pgpro560.py @@ -0,0 +1,98 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'pgpro560' + + +class CheckSystemID(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro560_control_file_loss(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node with stream support, delete control file + make backup + check that backup failed + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + file = os.path.join(node.base_dir,'data', 'global', 'pg_control') + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: could not open file' in e.message + and 'pg_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_pgpro560_systemid_mismatch(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node1 and node2 + feed to backup PGDATA from node1 and PGPORT from node2 + check that backup failed + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node1.start() + node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node2.start() + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + + try: + self.backup_node(backup_dir, 'node1', node2, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node1', node2, data_dir=node1.data_dir, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589.py new file mode 100644 index 00000000..bd40f16d --- /dev/null +++ b/tests/pgpro589.py @@ -0,0 +1,80 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'pgpro589' + + +class ArchiveCheck(ProbackupTest, unittest.TestCase): + + def test_pgpro589(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check that backup status equal to ERROR + check that no files where copied to backup catalogue + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + # make erroneus archive_command + node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").rstrip().decode( + "utf-8") + + try: + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing archive wal " + "segment with start_lsn.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for WAL segment' in e.message and + 'ERROR: Switched WAL segment' in e.message and + 'could not be archived' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup should have ERROR status') + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', path) + self.assertFalse( + os.path.isfile(file), + "\n Start LSN was not found in archive but datafiles where " + "copied to backup catalogue.\n For example: {0}\n " + "It is not optimal".format(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack.py b/tests/ptrack.py new file mode 100644 index 00000000..c2d6abff --- /dev/null +++ b/tests/ptrack.py @@ -0,0 +1,1600 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from testgres import QueryException +import shutil +import sys +import time + + +module_name = 'ptrack' + + +class PtrackTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_enable(self): + """make ptrack without full backup, should result in error""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack disabled.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: Ptrack is disabled\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_disable(self): + """ + Take full backup, disable ptrack restart postgresql, + enable ptrack, restart postgresql, take ptrack backup + which should fail + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # DISABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack_enable to off") + node.restart() + + # ENABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack_enable to on") + node.restart() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack_enable was set to OFF at some" + " point after previous backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: LSN from ptrack_control', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd + ) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_uncommited_xact(self): + """make ptrack backup while there is uncommited open transaction""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + self.backup_node(backup_dir, 'node', node) + con = node.connect("postgres") + con.execute( + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + pgdata = self.pgdata_content(node.data_dir) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_full(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + # create async connection + conn = self.get_async_connect(port=node.port) + + self.wait(conn) + + acurs = conn.cursor() + acurs.execute("select pg_backend_pid()") + + self.wait(conn) + pid = acurs.fetchall()[0][0] + print(pid) + + gdb = self.gdb_attach(pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + if not gdb.continue_execution_until_running(): + print('Failed gdb continue') + exit(1) + + acurs.execute("VACUUM FULL t_heap") + + if gdb.stopped_in_breakpoint(): + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_truncate(self): + """make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take ptrack backup, take second ptrack backup, + restore last ptrack backup and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, + ignore_ptrack=False + ) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_simple(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Logical comparison + self.assertEqual( + result, + node_restored.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_get_block(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('make_pagemap_from_ptrack') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_stream(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql("postgres", "create sequence t_seq") + node.safe_psql( + "postgres", + "create table t_heap as select i as id, nextval('t_seq')" + " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" + " as tsvector from generate_series(0,100) i" + ) + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, nextval('t_seq') as t_seq," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', + node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=["-j", "4", "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=["-j", "4", "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_archive(self): + """make archive node, make full and ptrack backups, + check data correctness in restored instance""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as" + " select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node(backup_dir, 'node', node) + full_target_time = self.show_pb( + backup_dir, 'node', full_backup_id)['recovery-time'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack') + ptrack_target_time = self.show_pb( + backup_dir, 'node', ptrack_backup_id)['recovery-time'] + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--recovery-target-action=promote", + "--time={0}".format(full_target_time)] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=[ + "-j", "4", + "--time={0}".format(ptrack_target_time), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_pgpro417(self): + """Make node, take full backup, take ptrack backup, + delete ptrack backup. Try to take ptrack backup, + which should fail""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': + 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql( + "postgres", + "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["--stream"]) + + start_lsn_full = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + start_lsn_ptrack = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + self.delete_pb(backup_dir, 'node', backup_id) + + # SECOND PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_pgpro417(self): + """ + Make archive node, take full backup, take page backup, + delete page backup. Try to take ptrack backup, which should fail + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.delete_pb(backup_dir, 'node', backup_id) +# sys.exit(1) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0}\n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_full_pgpro417(self): + """ + Make node, take two full backups, delete full second backup. + Try to take ptrack backup, which should fail + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector " + " from generate_series(0,100) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # SECOND FULL BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + + self.delete_pb(backup_dir, 'node', backup_id) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertTrue( + "ERROR: LSN from ptrack_control" in e.message and + "Create new full backup before " + "an incremental one" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take ptrack backup, + restore database and check it presense + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream", "--log-level-file=verbose"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND PTRACK BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_ptrack(self): + """Make node, create tablespace with table, take full backup, + alter tablespace location, take ptrack backup, restore database.""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new" + ) + + # sys.exit(1) + # PTRACK BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + # node.stop() + # node.cleanup() + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ), + "--recovery-target-action=promote" + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_ptrack(self): + """Make node, create tablespace with database," + " take full backup, alter tablespace location," + " take ptrack backup, restore database.""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE TABLESPACE + self.create_tblspace_in_node(node, 'somedata') + + # ALTER DATABASE + node.safe_psql( + "template1", + "alter database postgres set tablespace somedata") + + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", '--log-level-file=verbose']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + node.stop() + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', + node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata'))]) + + # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_drop_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to tablespace 'somedata' + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # Move table back to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # DROP TABLESPACE 'somedata' + node.safe_psql( + "postgres", "drop tablespace somedata") + # THIRD PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + tblspace = self.get_tblspace_path(node, 'somedata') + node.cleanup() + shutil.rmtree(tblspace, ignore_errors=True) + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + node.start() + + tblspc_exist = node.safe_psql( + "postgres", + "select exists(select 1 from " + "pg_tablespace where spcname = 'somedata')") + + if tblspc_exist.rstrip() == 't': + self.assertEqual( + 1, 0, + "Expecting Error because " + "tablespace 'somedata' should not be present") + + result_new = node.safe_psql("postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_alter_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + tblspc_path = self.get_tblspace_path(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to separate tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from t_heap") + + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"]) + + # GET PHYSICAL CONTENT FROM NODE + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore ptrack backup + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + # COMPARE LOGICAL CONTENT + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + restored_node.cleanup() + shutil.rmtree(tblspc_path_new, ignore_errors=True) + + # Move table to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore second ptrack backup and check table consistency + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_multiple_segments(self): + """ + Make node, create table, alter table tablespace, + take ptrack backup, move table from tablespace, take ptrack backup + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'ptrack_enable': 'on', 'fsync': 'off', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PTRACK STUFF + idx_ptrack = {'type': 'heap'} + idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') + idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') + idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], idx_ptrack['old_size']) + + pgbench = node.pgbench(options=['-T', '150', '-c', '2', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + idx_ptrack['new_size'] = self.get_fork_size( + node, + 'pgbench_accounts' + ) + idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], + idx_ptrack['new_size'] + ) + idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, + idx_ptrack['path'] + ) + self.check_ptrack_sanity(idx_ptrack) + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--log-level-file=verbose"] + ) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, + 'somedata_restored' + ) + + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", + "select * from pgbench_accounts" + ) + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_atexit_fail(self): + """ + Take backups of every available types and check that PTRACK is clean + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'max_connections': '15'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=[ + "--stream", "-j 30", + "--log-level-file=verbose"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are opening too many connections" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'setting its status to ERROR', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + self.assertEqual( + node.safe_psql( + "postgres", + "select * from pg_is_in_backup()").rstrip(), + "f") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py new file mode 100644 index 00000000..f4350af0 --- /dev/null +++ b/tests/ptrack_clean.py @@ -0,0 +1,253 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import time + + +module_name = 'ptrack_clean' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean(self): + """Take backups of every available types and check that PTRACK is clean""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata " + "as select i as id, nextval('t_seq') as t_seq, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + options=['-j10', '--stream']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['-j10', '--log-level-file=verbose']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['-j10']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean_replica(self): + """Take backups of every available types from master and check that PTRACK on replica is clean""" + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='ptrack', + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='page', + options=[ + '-j10', '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py new file mode 100644 index 00000000..784751ef --- /dev/null +++ b/tests/ptrack_cluster.py @@ -0,0 +1,268 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from time import sleep +from sys import exit + + +module_name = 'ptrack_cluster' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_cluster_on_btree(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_btree') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_gist(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_gist') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_btree_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_btree') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + #@unittest.skip("skip") + def test_ptrack_cluster_on_gist_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_gist') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py new file mode 100644 index 00000000..98c20914 --- /dev/null +++ b/tests/ptrack_move_to_tablespace.py @@ -0,0 +1,57 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_move_to_tablespace' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql("postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # Move table and indexes and make checkpoint + for i in idx_ptrack: + if idx_ptrack[i]['type'] == 'heap': + node.safe_psql('postgres', 'alter table {0} set tablespace somedata;'.format(i)) + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql('postgres', 'alter index {0} set tablespace somedata'.format(i)) + node.safe_psql('postgres', 'checkpoint') + + # Check ptrack files + for i in idx_ptrack: + if idx_ptrack[i]['type'] == 'seq': + continue + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py new file mode 100644 index 00000000..8569ef59 --- /dev/null +++ b/tests/ptrack_recovery.py @@ -0,0 +1,58 @@ +import os +import unittest +from sys import exit +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_recovery' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table + node.safe_psql("postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = int(self.get_fork_size(node, i)) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + if not node.status(): + node.start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + for i in idx_ptrack: + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py new file mode 100644 index 00000000..928608c4 --- /dev/null +++ b/tests/ptrack_truncate.py @@ -0,0 +1,130 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_truncate' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'truncate t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + replica.safe_psql('postgres', 'truncate t_heap') + replica.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py new file mode 100644 index 00000000..0409cae3 --- /dev/null +++ b/tests/ptrack_vacuum.py @@ -0,0 +1,152 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make FULL backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py new file mode 100644 index 00000000..f0cd3bbd --- /dev/null +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -0,0 +1,136 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_bits_frozen' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_frozen(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum freeze t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_bits_frozen_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take PTRACK backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'vacuum freeze t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py new file mode 100644 index 00000000..45a8d9b6 --- /dev/null +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -0,0 +1,67 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_bits_visibility' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_visibility(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py new file mode 100644 index 00000000..ec12c9e2 --- /dev/null +++ b/tests/ptrack_vacuum_full.py @@ -0,0 +1,140 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_full' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum full t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take FULL backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum full t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py new file mode 100644 index 00000000..5c84c7e8 --- /dev/null +++ b/tests/ptrack_vacuum_truncate.py @@ -0,0 +1,142 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_truncate' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id > 128;') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take PTRACK backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id > 128;') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/replica.py b/tests/replica.py new file mode 100644 index 00000000..d74c375c --- /dev/null +++ b/tests/replica.py @@ -0,0 +1,293 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess +from sys import exit +import time + + +module_name = 'replica' + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_ptrack_backup(self): + """ + make node, take full backup, restore it and make replica from it, + take full stream backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} + ) + master.start() + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + # CREATE TABLE + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # take full backup and restore it + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica) + + # Check data correctness on replica + replica.slow_start(replica=True) + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take PTRACK backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PTRACK BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_archive_page_backup(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data + # equal to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=300', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + options=[ + '--archive-timeout=300', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_make_replica_via_restore(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node( + backup_dir, 'master', replica, + options=['-R', '--recovery-target-action=promote']) + + # Settings for Replica + # self.set_replica(master, replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/restore_test.py b/tests/restore_test.py new file mode 100644 index 00000000..c33a1e29 --- /dev/null +++ b/tests/restore_test.py @@ -0,0 +1,1243 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from datetime import datetime +import sys +import time + + +module_name = 'restore' + + +class RestoreTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_restore_full_to_latest(self): + """recovery to latest from full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # 1 - Test recovery from latest + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # 2 - Test that recovery.conf was created + recovery_conf = os.path.join(node.data_dir, "recovery.conf") + self.assertEqual(os.path.isfile(recovery_conf), True) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_page_to_latest(self): + """recovery to latest from full + page backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_specific_timeline(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_tli = int( + node.get_control_data()["Latest checkpoint's TimeLineID"]) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # Correct Backup must be choosen for restore + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--timeline={0}".format(target_tli), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + recovery_target_timeline = self.get_recovery_conf( + node)["recovery_target_timeline"] + self.assertEqual(int(recovery_target_timeline), target_tli) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_time(self): + """recovery to target time""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.append_conf("postgresql.auto.conf", "TimeZone = Europe/Moscow") + node.start() + + node.pgbench_init(scale=2) + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(target_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_xid_inclusive(self): + """recovery to target xid""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--xid={0}'.format(target_xid), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_xid_not_inclusive(self): + """recovery with target inclusive false""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = result[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + '--xid={0}'.format(target_xid), + "--inclusive=false", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_not_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "--inclusive=false", + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_archive(self): + """recovery to latest from archive full+ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_ptrack(self): + """recovery to latest from archive full+ptrack+ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_stream(self): + """recovery in stream mode to latest from full + ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_under_load(self): + """ + recovery to latest from full + ptrack backups + with loads when ptrack backup do + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + pgbench.wait() + pgbench.stdout.close() + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_under_load_ptrack(self): + """ + recovery to latest from full + page backups + with loads when full backup do + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # wal_segment_size = self.guc_wal_segment_size(node) + node.pgbench_init(scale=2) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + self.backup_node(backup_dir, 'node', node) + + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + + node.stop() + node.cleanup() + # self.wrong_wal_clean(node, wal_segment_size) + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_1(self): + """recovery using tablespace-mapping option""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") + con.execute("INSERT INTO test VALUES (1)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # 1 - Try to restore to existing directory + node.stop() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore destionation is not empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: restore destination is not empty: "{0}"\n'.format( + node.data_dir), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 2 - Try to restore to existing tablespace directory + node.cleanup() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore tablespace destination is " + "not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: restore tablespace destination ' + 'is not empty: "{0}"\n'.format(tblspc_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 3 - Restore using tablespace-mapping + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(result[0][0], 1) + + # 4 - Restore using tablespace-mapping using page backup + self.backup_node(backup_dir, 'node', node) + with node.connect("postgres") as con: + con.execute("INSERT INTO test VALUES (2)") + con.commit() + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + show_pb = self.show_pb(backup_dir, 'node') + self.assertEqual(show_pb[1]['status'], "OK") + self.assertEqual(show_pb[2]['status'], "OK") + + node.stop() + node.cleanup() + tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page), + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(result[0][0], 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_2(self): + """recovery using tablespace-mapping option and page backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Full backup + self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # First page backup + self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") + + # Create tablespace table + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CHECKPOINT") + con.connection.autocommit = False + con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") + con.execute( + "INSERT INTO tbl1 SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Second page backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") + + node.stop() + node.cleanup() + + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + count = node.execute("postgres", "SELECT count(*) FROM tbl1") + self.assertEqual(count[0][0], 4) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.safe_psql("postgres", "select pg_switch_xlog()") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_pitr(self): + """ + make node with archiving, make stream backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_archive_pitr_2(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_restore_to_restore_point(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,10000)") + result = node.safe_psql( + "postgres", + "select * from t_heap") + node.safe_psql( + "postgres", "select pg_create_restore_point('savepoint')") + node.safe_psql( + "postgres", + "create table t_heap_1 as select generate_series(0,10000)") + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + "--recovery-target-name=savepoint", + "--recovery-target-action=promote"]) + + node.slow_start() + + result_new = node.safe_psql("postgres", "select * from t_heap") + res = node.psql("postgres", "select * from t_heap_1") + self.assertEqual( + res[0], 1, + "Table t_heap_1 should not exist in restored instance") + + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/retention_test.py b/tests/retention_test.py new file mode 100644 index 00000000..652f7c39 --- /dev/null +++ b/tests/retention_test.py @@ -0,0 +1,178 @@ +import os +import unittest +from datetime import datetime, timedelta +from .helpers.ptrack_helpers import ProbackupTest + + +module_name = 'retention' + + +class RetentionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_retention_redundancy_1(self): + """purge backups using redundancy-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backups to be keeped + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + log = self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Check that WAL segments were deleted + min_wal = None + max_wal = None + for line in log.splitlines(): + if line.startswith("INFO: removed min WAL segment"): + min_wal = line[31:-1] + elif line.startswith("INFO: removed max WAL segment"): + max_wal = line[31:-1] + + if not min_wal: + self.assertTrue(False, "min_wal is empty") + + if not max_wal: + self.assertTrue(False, "max_wal is not set") + + for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): + if not wal_name.endswith(".backup"): + # wal_name_b = wal_name.encode('ascii') + self.assertEqual(wal_name[8:] > min_wal[8:], True) + self.assertEqual(wal_name[8:] > max_wal[8:], True) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# @unittest.skip("123") + def test_retention_window_2(self): + """purge backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with open( + os.path.join( + backup_dir, + 'backups', + 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + conf.write("retention-window = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# @unittest.skip("123") + def test_retention_wal(self): + """purge backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + # Take FULL BACKUP + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--retention-window=2']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/show_test.py b/tests/show_test.py new file mode 100644 index 00000000..931da184 --- /dev/null +++ b/tests/show_test.py @@ -0,0 +1,203 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'show' + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_1(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_json(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_corrupt_2(self): + """Status CORRUPT""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete file which belong to backup + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node', backup_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup corrupted.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'data files are corrupted\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_no_control_file(self): + """backup.control doesn't exist""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + os.remove(file) + + self.assertIn('control file "{0}" doesn\'t exist'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_empty_control_file(self): + """backup.control is empty""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # truncate backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'w') + fd.close() + + self.assertIn('control file "{0}" is empty'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_control_file(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # corrupt backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'a') + fd.write("statuss = OK") + fd.close() + + self.assertIn('invalid option "statuss" in file'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py new file mode 100644 index 00000000..ab091c57 --- /dev/null +++ b/tests/validate_test.py @@ -0,0 +1,1730 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from sys import exit +import time + + +module_name = 'validate' + + +class ValidateTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_validate_wal_unreal_values(self): + """ + make node with archiving, make archive backup + validate to both real and unreal values + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + + pgbench.wait() + pgbench.stdout.close() + + target_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + after_backup_time = datetime.now().replace(second=0, microsecond=0) + + # Validate to real time + self.assertIn( + "INFO: backup validation completed successfully", + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(target_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Validate to unreal time + unreal_time_1 = after_backup_time - timedelta(days=2) + try: + self.validate_pb( + backup_dir, 'node', options=["--time={0}".format( + unreal_time_1)]) + self.assertEqual( + 1, 0, + "Expecting Error because of validation to unreal time.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Full backup satisfying target options is not found.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Validate to unreal time #2 + unreal_time_2 = after_backup_time + timedelta(days=2) + try: + self.validate_pb(backup_dir, 'node', options=["--time={0}".format(unreal_time_2)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('ERROR: not enough WAL records to time' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Validate to real xid + target_xid = None + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + self.switch_wal_segment(node) + + self.assertIn("INFO: backup validation completed successfully", + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Validate to unreal xid + unreal_xid = int(target_xid) + 1000 + try: + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('ERROR: not enough WAL records to xid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Validate with backup ID + self.assertIn("INFO: Validating backup {0}".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} data files are valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} WAL segments are valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} is valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Validate of backup {0} completed".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backup(self): + """make archive node, take FULL, PAGE1, PAGE2 backups, corrupt file in PAGE1 backup, + run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") + # PAGE2 + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file + file = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path) + with open(file, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id_2, + options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format(backup_id_2) in e.message + and 'ERROR: Backup {0} is corrupt'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups(self): + """make archive node, take FULL, PAGE1, PAGE2 backups, + corrupt file in FULL and PAGE1 backupd, run validate on PAGE1, + expect FULL and PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap_1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Corrupt some file in PAGE1 backup + file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path_t_heap_1) + with open(file_page1, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE1 + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id_2, + options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('INFO: Validating parents for backup {0}'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent'.format(backup_id_2) in e.message + and 'WARNING: Backup {0} is orphaned because his parent'.format(backup_id_3) in e.message + and 'ERROR: Backup {0} is orphan.'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups_1(self): + """make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 backups, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + backup_id_4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE4 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + backup_id_6 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_7 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join( + backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join( + backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb( + backup_dir, 'node', + backup_id=backup_id_4, + options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_page1) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_specific_target_corrupted_intermediate_backups(self): + """make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 backups, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3 to specific xid, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") + backup_id_4 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE4 + target_xid = node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i RETURNING (xmin)")[0][0] + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + backup_id_6 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_7 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join(backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb(backup_dir, 'node', + options=['--log-level-file=verbose', '-i', backup_id_4, '--xid={0}'.format(target_xid)]) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and 'INFO: Backup {0} data files are valid'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_2) in e.message + and 'INFO: Backup {0} data files are valid'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_3) in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_page1) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_page(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in PAGE1 backup and run validate on instance, + expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + # FULL1 + backup_id_4 = self.backup_node( + backup_dir, 'node', node) + # PAGE3 + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join( + backup_dir, 'backups/node', backup_id_2, + 'database', file_path_t_heap1) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb( + backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message, + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_5) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_3) in e.message and + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_3, backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_full) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Some backups are not valid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full_and_try_restore(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, + try to restore backup with --no-validation option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + node.cleanup() + restore_out = self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id_5), + restore_out, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_1(self): + """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(42) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_2(self): + """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + target_xid = None + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(128) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_1(self): + """make archive node, make archive full backup, + delete from archive wal segment which belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'CORRUPT' + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + backup_id = self.backup_node(backup_dir, 'node', node) + + # Delete wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + file = os.path.join(backup_dir, 'wal', 'node', wals[-1]) + os.remove(file) + + # cut out '.gz' + if self.archive_compress: + file = file[:-3] + + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: WAL segment \"{0}\" is absent".format( + file) in e.message and + "WARNING: There are not enough WAL records to consistenly " + "restore backup {0}".format(backup_id) in e.message and + "WARNING: Backup {0} WAL segments are corrupted".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup {0} should have STATUS "CORRUPT"') + + # Run validate again + try: + self.validate_pb(backup_dir, 'node', backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'INFO: Revalidating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Backup {0} is corrupt.'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_between_backups(self): + """ + make archive node, make full backup, corrupt all wal files, + run validate to real xid, expect errors + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + if self.get_version(node) < self.version_to_num('10.0'): + walfile = node.safe_psql( + 'postgres', + 'select pg_xlogfile_name(pg_current_xlog_location())').rstrip() + else: + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() + + if self.archive_compress: + walfile = walfile + '.gz' + self.switch_wal_segment(node) + + # generate some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: + f.seek(9000) + f.write(b"b") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: not enough WAL records to xid' in e.message and + 'WARNING: recovery can be done up to time' in e.message and + "ERROR: not enough WAL records to xid {0}\n".format( + target_xid), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_2(self): + """ + make node with archiving + make archive backup + delete from archive wal segment which DO NOT belong to this backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'ERROR' + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( + wals_dir, f)) and not f.endswith('.backup')] + wals = map(str, wals) + file = os.path.join(wals_dir, max(wals)) + os.remove(file) + if self.archive_compress: + file = file[:-3] + + # Try to restore + try: + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for LSN' in e.message and + 'in archived WAL segment' in e.message and + 'WARNING: could not read WAL record at' in e.message and + 'ERROR: WAL segment "{0}" is absent\n'.format( + file) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro702_688(self): + """make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + recovery_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + try: + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(recovery_time)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WAL archive is empty. You cannot restore backup to a ' + 'recovery target without WAL archive', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro688(self): + """make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + + self.validate_pb(backup_dir, 'node', options=["--time={0}".format(recovery_time)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro561(self): + """ + make node with archiving, make stream backup, + restore it to node1, check that archiving is not successful on node1 + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node( + base_dir="{0}/{1}/node1".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + self.set_archiving(backup_dir, 'node1', node1) + node1.start() + + backup_id = self.backup_node( + backup_dir, 'node1', node1, options=["--stream"]) + + node2 = self.make_simple_node( + base_dir="{0}/{1}/node2".format(module_name, fname)) + node2.cleanup() + + node1.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node( + backup_dir, 'node1', node1, + backup_type='page', options=["--stream"]) + self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + node2.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) + node2.slow_start() + + timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] + timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual( + timeline_node1, timeline_node2, + "Timelines on Master and Node1 should be equal. " + "This is unexpected") + + archive_command_node1 = node1.safe_psql( + "postgres", "show archive_command") + archive_command_node2 = node2.safe_psql( + "postgres", "show archive_command") + self.assertEqual( + archive_command_node1, archive_command_node2, + "Archive command on Master and Node should be equal. " + "This is unexpected") + + # result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + ## self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) + # if result == "": + # self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') + + self.switch_wal_segment(node1) + self.switch_wal_segment(node2) + time.sleep(5) + + log_file = os.path.join(node2.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse( + 'pg_probackup archive-push completed successfully' in log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_full(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and three page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption and run valudate again, check that + second full backup and his page backups are OK + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "alter system set archive_command = 'false'") + node.reload() + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=1s']) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + pass + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.set_archiving(backup_dir, 'node', node) + node.reload() + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue( + self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue( + self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + + os.rename(file_new, file) + try: + self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_full_1(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and four page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption from full backup and corrupt his second page backup + run valudate again, check that + second full backup and his firts page backups are OK, + second page should be CORRUPT + third page should be ORPHAN + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_page = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + os.rename(file_new, file) + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id_page, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_file_size_corruption_no_validate(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + # initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4"], async=False, gdb=False) + + node.stop() + node.cleanup() + + # Let`s do file corruption + with open(os.path.join(backup_dir, "backups", 'node', backup_id, "database", heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + except ProbackupException as e: + self.assertTrue("ERROR: Data files restoring failed" in e.message, repr(e.message)) + print "\nExpected error: \n" + e.message + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/travis/backup_restore.sh b/travis/backup_restore.sh new file mode 100644 index 00000000..7fe1cfd8 --- /dev/null +++ b/travis/backup_restore.sh @@ -0,0 +1,66 @@ +#!/bin/sh -ex + +# vars +export PGVERSION=9.5.4 +export PATH=$PATH:/usr/pgsql-9.5/bin +export PGUSER=pgbench +export PGDATABASE=pgbench +export PGDATA=/var/lib/pgsql/9.5/data +export BACKUP_PATH=/backups +export ARCLOG_PATH=$BACKUP_PATH/backup/pg_xlog +export PGDATA2=/var/lib/pgsql/9.5/data2 +export PGBENCH_SCALE=100 +export PGBENCH_TIME=60 + +# prepare directory +cp -a /tests /build +pushd /build + +# download postgresql +yum install -y wget +wget -k https://ftp.postgresql.org/pub/source/v$PGVERSION/postgresql-$PGVERSION.tar.gz -O postgresql.tar.gz +tar xf postgresql.tar.gz + +# install pg_probackup +yum install -y https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-7-x86_64/pgdg-centos95-9.5-2.noarch.rpm +yum install -y postgresql95-devel make gcc readline-devel openssl-devel pam-devel libxml2-devel libxslt-devel +make top_srcdir=postgresql-$PGVERSION +make install top_srcdir=postgresql-$PGVERSION + +# initalize cluster and database +yum install -y postgresql95-server +su postgres -c "/usr/pgsql-9.5/bin/initdb -D $PGDATA -k" +cat < $PGDATA/pg_hba.conf +local all all trust +host all all 127.0.0.1/32 trust +local replication pgbench trust +host replication pgbench 127.0.0.1/32 trust +EOF +cat < $PGDATA/postgresql.auto.conf +max_wal_senders = 2 +wal_level = logical +wal_log_hints = on +EOF +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA" +su postgres -c "createdb -U postgres $PGUSER" +su postgres -c "createuser -U postgres -a -d -E $PGUSER" +pgbench -i -s $PGBENCH_SCALE + +# Count current +COUNT=$(psql -Atc "select count(*) from pgbench_accounts") +pgbench -s $PGBENCH_SCALE -T $PGBENCH_TIME -j 2 -c 10 & + +# create backup +pg_probackup init +pg_probackup backup -b full --disable-ptrack-clear --stream -v +pg_probackup show +sleep $PGBENCH_TIME + +# restore from backup +chown -R postgres:postgres $BACKUP_PATH +su postgres -c "pg_probackup restore -D $PGDATA2" + +# start backup server +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl stop -w -D $PGDATA" +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA2" +( psql -Atc "select count(*) from pgbench_accounts" | grep $COUNT ) || (cat $PGDATA2/pg_log/*.log ; exit 1) diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 00000000..14864181 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 00000000..c869e485 --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 00000000..a4f75553 --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + + From 078d341bb8990ec934d011314ef9f4a8162208aa Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Fri, 2 Nov 2018 15:47:15 +0300 Subject: [PATCH 09/37] pg_control: fix reading of pg_control of vanilla in pg_controldata Do not change the CRC as this makes its check senseless. Instead, read the pg_control of vanilla to the pg_control strcucture of vanilla (ControlFileDataOriginal) and print its values as is done in vanilla. Thanks to Arthur Zakirov for reporting this. Also: fix checking of byte ordering mismatch of pg_control in pg_probackup. --- src/util.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util.c b/src/util.c index 4eefa788..27a0ca24 100644 --- a/src/util.c +++ b/src/util.c @@ -71,7 +71,8 @@ checkControlFile(ControlFileData *ControlFile) "Either the file is corrupt, or it has a different layout than this program\n" "is expecting. The results below are untrustworthy."); - if (ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0) + if ((ControlFile->pg_control_version % 65536 == 0 || ControlFile->pg_control_version % 65536 > 10000) && + ControlFile->pg_control_version / 65536 != 0) elog(ERROR, "possible byte ordering mismatch\n" "The byte ordering used to store the pg_control file might not match the one\n" "used by this program. In that case the results below would be incorrect, and\n" From 6c9dfcfe8290df7745356ac9183d82dca61ade98 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 7 Nov 2018 04:21:56 +0300 Subject: [PATCH 10/37] PGPRO-2095: use latest replayed lsn instead of STOP LSN --- src/backup.c | 80 +++++++++++++++++++++++++++------------------- src/pg_probackup.h | 4 +++ src/util.c | 30 ++++++++--------- 3 files changed, 67 insertions(+), 47 deletions(-) diff --git a/src/backup.c b/src/backup.c index 602ab823..e3e1d60a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -756,28 +756,25 @@ do_backup_instance(void) parray_free(prev_backup_filelist); } - /* Copy pg_control in case of backup from replica >= 9.6 */ + /* In case of backup from replica >= 9.6 we must fix minRecPoint, + * First we must find pg_control in backup_files_list. + */ if (current.from_replica && !exclusive_backup) { + char pg_control_path[MAXPGPATH]; + + snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s", pgdata, "global/pg_control"); + for (i = 0; i < parray_num(backup_files_list); i++) { pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); - if (strcmp(tmp_file->name, "pg_control") == 0) + if (strcmp(tmp_file->path, pg_control_path) == 0) { pg_control = tmp_file; break; } } - - if (!pg_control) - elog(ERROR, "Failed to locate pg_control in copied files"); - - if (is_remote_backup) - remote_copy_file(NULL, pg_control); - else - if (!copy_file(pgdata, database_path, pg_control)) - elog(ERROR, "Failed to copy pg_control"); } @@ -1160,9 +1157,6 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) */ pg_switch_wal(conn); - //elog(INFO, "START LSN: %X/%X", - // (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn)); - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) /* In PAGE mode wait for current segment... */ wait_wal_lsn(backup->start_lsn, true, false); @@ -1175,8 +1169,10 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) /* ...for others wait for previous segment */ wait_wal_lsn(backup->start_lsn, true, true); - /* Wait for start_lsn to be replayed by replica */ - if (backup->from_replica) + /* In case of backup from replica for PostgreSQL 9.5 + * wait for start_lsn to be replayed by replica + */ + if (backup->from_replica && exclusive_backup) wait_replica_wal_lsn(backup->start_lsn, true); } @@ -1526,7 +1522,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size); /* - * In pg_start_backup we wait for 'lsn' in 'pg_wal' directory iff it is + * In pg_start_backup we wait for 'lsn' in 'pg_wal' directory if it is * stream and non-page backup. Page backup needs archived WAL files, so we * wait for 'lsn' in archive 'wal' directory for page backups. * @@ -1547,7 +1543,12 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) { join_path_components(wal_segment_path, arclog_path, wal_segment); wal_segment_dir = arclog_path; - timeout = archive_timeout; + + if (archive_timeout > 0) + timeout = archive_timeout; + else + timeout = ARCHIVE_TIMEOUT_DEFAULT; + } if (wait_prev_segment) @@ -1780,14 +1781,29 @@ pg_stop_backup(pgBackup *backup) * Stop the non-exclusive backup. Besides stop_lsn it returns from * pg_stop_backup(false) copy of the backup label and tablespace map * so they can be written to disk by the caller. + * In case of backup from replica >= 9.6 we do not trust minRecPoint + * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. */ - stop_backup_query = "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " lsn," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)"; + if (current.from_replica) + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," +#if PG_VERSION_NUM >= 100000 + " pg_catalog.pg_last_wal_replay_lsn()," +#else + " pg_catalog.pg_last_xlog_replay_location()," +#endif + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; + else + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; } else @@ -1873,14 +1889,14 @@ pg_stop_backup(pgBackup *backup) /* Calculate LSN */ stop_backup_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - //if (!XRecOffIsValid(stop_backup_lsn)) - //{ - // stop_backup_lsn = restore_lsn; - //} - if (!XRecOffIsValid(stop_backup_lsn)) - elog(ERROR, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + { + if (XRecOffIsNull(stop_backup_lsn)) + stop_backup_lsn = stop_backup_lsn + SizeOfXLogLongPHD; + else + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + } /* Write backup_label and tablespace_map */ if (!exclusive_backup) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b75bb581..f5d6bb5c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -57,6 +57,10 @@ #define XID_FMT "%u" #endif +/* Check if an XLogRecPtr value is pointed to 0 offset */ +#define XRecOffIsNull(xlrp) \ + ((xlrp) % XLOG_BLCKSZ == 0) + typedef enum CompressAlg { NOT_DEFINED_COMPRESS = 0, diff --git a/src/util.c b/src/util.c index 5f059c37..e20cda17 100644 --- a/src/util.c +++ b/src/util.c @@ -119,7 +119,7 @@ writeControlFile(ControlFileData *ControlFile, char *path) /* copy controlFileSize */ buffer = pg_malloc(ControlFileSize); - memcpy(buffer, &ControlFile, sizeof(ControlFileData)); + memcpy(buffer, ControlFile, sizeof(ControlFileData)); /* Write pg_control */ unlink(path); @@ -136,8 +136,8 @@ writeControlFile(ControlFileData *ControlFile, char *path) if (fsync(fd) != 0) elog(ERROR, "Failed to fsync file: %s", path); - pg_free(buffer); close(fd); + pg_free(buffer); } /* @@ -290,9 +290,7 @@ get_data_checksum_version(bool safe) return ControlFile.data_checksum_version; } -/* MinRecoveryPoint 'as-is' is not to be trusted - * Use STOP LSN instead - */ +/* MinRecoveryPoint 'as-is' is not to be trusted */ void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn) { @@ -301,20 +299,21 @@ set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_ba size_t size; char fullpath[MAXPGPATH]; - elog(LOG, "Setting minRecPoint to STOP LSN: %X/%X", - (uint32) (stop_backup_lsn >> 32), - (uint32) stop_backup_lsn); - - /* Path to pg_control in backup */ - snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); - - /* First fetch file... */ - buffer = slurpFile(backup_path, XLOG_CONTROL_FILE, &size, false); + /* First fetch file content */ + buffer = slurpFile(pgdata, XLOG_CONTROL_FILE, &size, false); if (buffer == NULL) elog(ERROR, "ERROR"); digestControlFile(&ControlFile, buffer, size); + elog(LOG, "Current minRecPoint %X/%X", + (uint32) (ControlFile.minRecoveryPoint >> 32), + (uint32) ControlFile.minRecoveryPoint); + + elog(LOG, "Setting minRecPoint to %X/%X", + (uint32) (stop_backup_lsn >> 32), + (uint32) stop_backup_lsn); + ControlFile.minRecoveryPoint = stop_backup_lsn; /* Update checksum in pg_control header */ @@ -327,7 +326,8 @@ set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_ba /* paranoia */ checkControlFile(&ControlFile); - /* update pg_control */ + /* overwrite pg_control */ + snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); writeControlFile(&ControlFile, fullpath); /* Update pg_control checksum in backup_list */ From 5e12fec6ab34b0b031b9f380dd63ab884d8657a2 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 8 Nov 2018 12:43:19 +0300 Subject: [PATCH 11/37] Eliminate unsued varaibles --- src/data.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 876e7f1e..a8378f09 100644 --- a/src/data.c +++ b/src/data.c @@ -127,8 +127,7 @@ page_may_be_compressed(Page page, CompressAlg alg) phdr->pd_special == MAXALIGN(phdr->pd_special))) { /* ... end only if it is invalid, then do more checks */ - int major, middle, minor; - if ( parse_program_version(current.program_version) >= 20023) + if (parse_program_version(current.program_version) >= 20023) { /* Versions 2.0.23 and higher don't have such bug */ return false; From ea3ed9fa171f5807bdb4790f8b64a0211cb729a7 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 8 Nov 2018 15:04:55 +0300 Subject: [PATCH 12/37] Instead of current global backup variable use function local argument, which is sent by callers --- src/data.c | 59 +++++++++++++++++++++++-------- src/merge.c | 9 +++-- src/pg_probackup.h | 7 ++-- src/restore.c | 3 +- src/validate.c | 11 +++--- tests/expected/option_version.out | 2 +- 6 files changed, 63 insertions(+), 28 deletions(-) diff --git a/src/data.c b/src/data.c index a8378f09..5edeefa4 100644 --- a/src/data.c +++ b/src/data.c @@ -57,7 +57,7 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) */ static int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, - CompressAlg alg, int level) + CompressAlg alg, int level, const char **errormsg) { switch (alg) { @@ -66,7 +66,13 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - return zlib_compress(dst, dst_size, src, src_size, level); + { + int32 ret; + ret = zlib_compress(dst, dst_size, src, src_size, level); + if (ret != Z_OK && errormsg) + *errormsg = zError(ret); + return ret; + } #endif case PGLZ_COMPRESS: return pglz_compress(src, src_size, dst, PGLZ_strategy_always); @@ -81,7 +87,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, */ static int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, - CompressAlg alg) + CompressAlg alg, const char **errormsg) { switch (alg) { @@ -90,7 +96,13 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - return zlib_decompress(dst, dst_size, src, src_size); + { + int32 ret; + ret = zlib_decompress(dst, dst_size, src, src_size); + if (ret != Z_OK && errormsg) + *errormsg = zError(ret); + return ret; + } #endif case PGLZ_COMPRESS: return pglz_decompress(src, src_size, dst, dst_size); @@ -110,7 +122,7 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, * But at least we will do this check only for pages which will no pass validation step. */ static bool -page_may_be_compressed(Page page, CompressAlg alg) +page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) { PageHeader phdr; @@ -127,7 +139,7 @@ page_may_be_compressed(Page page, CompressAlg alg) phdr->pd_special == MAXALIGN(phdr->pd_special))) { /* ... end only if it is invalid, then do more checks */ - if (parse_program_version(current.program_version) >= 20023) + if (backup_version >= 20023) { /* Versions 2.0.23 and higher don't have such bug */ return false; @@ -434,9 +446,16 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, } else { + const char *errormsg = NULL; + /* The page was not truncated, so we need to compress it */ header.compressed_size = do_compress(compressed_page, BLCKSZ, - page, BLCKSZ, calg, clevel); + page, BLCKSZ, calg, clevel, + &errormsg); + /* Something went wrong and errormsg was assigned, throw a warning */ + if (header.compressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during compressing block %u of file \"%s\": %s", + blknum, file->path, errormsg); file->compress_alg = calg; file->read_size += BLCKSZ; @@ -473,7 +492,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, fclose(in); fclose(out); - elog(ERROR, "File: %s, cannot write backup at block %u : %s", + elog(ERROR, "File: %s, cannot write backup at block %u: %s", file->path, blknum, strerror(errno_tmp)); } @@ -661,7 +680,7 @@ backup_data_file(backup_files_arg* arguments, */ void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, - bool write_header) + bool write_header, uint32 backup_version) { FILE *in = NULL; FILE *out = NULL; @@ -766,14 +785,19 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, read_len, header.compressed_size); if (header.compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg)) + || page_may_be_compressed(compressed_page.data, file->compress_alg, + backup_version)) { int32 uncompressed_size = 0; + const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, compressed_page.data, header.compressed_size, - file->compress_alg); + file->compress_alg, &errormsg); + if (uncompressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", + blknum, file->path, errormsg); if (uncompressed_size != BLCKSZ) elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", @@ -1578,7 +1602,8 @@ validate_one_page(Page page, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool -check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version) +check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, + uint32 backup_version) { size_t read_len = 0; bool is_valid = true; @@ -1645,14 +1670,20 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version) blknum, file->path, read_len, header.compressed_size); if (header.compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg)) + || page_may_be_compressed(compressed_page.data, file->compress_alg, + backup_version)) { int32 uncompressed_size = 0; + const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, compressed_page.data, header.compressed_size, - file->compress_alg); + file->compress_alg, + &errormsg); + if (uncompressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", + blknum, file->path, errormsg); if (uncompressed_size != BLCKSZ) { diff --git a/src/merge.c b/src/merge.c index 2464199f..137f1acd 100644 --- a/src/merge.c +++ b/src/merge.c @@ -457,14 +457,16 @@ merge_files(void *arg) file->path = to_path_tmp; /* Decompress first/target file */ - restore_data_file(tmp_file_path, file, false, false); + restore_data_file(tmp_file_path, file, false, false, + parse_program_version(to_backup->program_version)); file->path = prev_path; } /* Merge second/source file with first/target file */ restore_data_file(tmp_file_path, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - false); + false, + parse_program_version(from_backup->program_version)); elog(VERBOSE, "Compress file and save it to the directory \"%s\"", argument->to_root); @@ -496,7 +498,8 @@ merge_files(void *arg) /* We can merge in-place here */ restore_data_file(to_path_tmp, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - true); + true, + parse_program_version(from_backup->program_version)); /* * We need to calculate write_size, restore_data_file() doesn't diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7bd87e56..182a647b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -517,7 +517,8 @@ extern bool backup_data_file(backup_files_arg* arguments, CompressAlg calg, int clevel); extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, - bool write_header); + bool write_header, + uint32 backup_version); extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, @@ -526,8 +527,8 @@ extern void get_wal_file(const char *from_path, const char *to_path); extern bool calc_file_checksum(pgFile *file); -extern bool check_file_pages(pgFile* file, - XLogRecPtr stop_lsn, uint32 checksum_version); +extern bool check_file_pages(pgFile* file, XLogRecPtr stop_lsn, + uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ extern void extractPageMap(const char *archivedir, diff --git a/src/restore.c b/src/restore.c index c08e647c..439f3c4e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -621,7 +621,8 @@ restore_files(void *arg) file->path + strlen(from_root) + 1); restore_data_file(to_path, file, arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - false); + false, + parse_program_version(arguments->backup->program_version)); } else copy_file(from_root, pgdata, file); diff --git a/src/validate.c b/src/validate.c index 4fa7d78b..5cbb9b26 100644 --- a/src/validate.c +++ b/src/validate.c @@ -28,6 +28,7 @@ typedef struct bool corrupted; XLogRecPtr stop_lsn; uint32 checksum_version; + uint32 backup_version; /* * Return value from the thread. @@ -92,11 +93,6 @@ pgBackupValidate(pgBackup *backup) pg_atomic_clear_flag(&file->lock); } - /* - * We use program version to calculate checksum in pgBackupValidateFiles() - */ - validate_backup_version = parse_program_version(backup->program_version); - /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (validate_files_arg *) @@ -111,6 +107,7 @@ pgBackupValidate(pgBackup *backup) arg->corrupted = false; arg->stop_lsn = backup->stop_lsn; arg->checksum_version = backup->checksum_version; + arg->backup_version = parse_program_version(backup->program_version); /* By default there are some error */ threads_args[i].ret = 1; @@ -233,7 +230,9 @@ pgBackupValidateFiles(void *arg) /* validate relation blocks */ if (file->is_datafile) { - if (!check_file_pages(file, arguments->stop_lsn, arguments->checksum_version)) + if (!check_file_pages(file, arguments->stop_lsn, + arguments->checksum_version, + arguments->backup_version)) arguments->corrupted = true; } } diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index cb0a30d4..6a0391c2 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.22 \ No newline at end of file +pg_probackup 2.0.23 \ No newline at end of file From 728e3d5bdc174c8c604c7ee8a57350b4c1847a5e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 8 Nov 2018 15:25:28 +0300 Subject: [PATCH 13/37] Avoid compression warnings for already compressed pages --- src/data.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 5edeefa4..d3e48e9a 100644 --- a/src/data.c +++ b/src/data.c @@ -449,7 +449,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, const char *errormsg = NULL; /* The page was not truncated, so we need to compress it */ - header.compressed_size = do_compress(compressed_page, BLCKSZ, + header.compressed_size = do_compress(compressed_page, sizeof(compressed_page), page, BLCKSZ, calg, clevel, &errormsg); /* Something went wrong and errormsg was assigned, throw a warning */ @@ -459,7 +459,6 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, file->compress_alg = calg; file->read_size += BLCKSZ; - Assert (header.compressed_size <= BLCKSZ); /* The page was successfully compressed. */ if (header.compressed_size > 0 && header.compressed_size < BLCKSZ) From 53c1a05b9b5406af0bcad81b8a8f87cf11794f75 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 8 Nov 2018 15:50:24 +0300 Subject: [PATCH 14/37] Remove unnecessary validate_backup_version --- src/validate.c | 4 +- src/validate.c.autosave | 532 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 533 insertions(+), 3 deletions(-) create mode 100644 src/validate.c.autosave diff --git a/src/validate.c b/src/validate.c index 5cbb9b26..2f75cd4b 100644 --- a/src/validate.c +++ b/src/validate.c @@ -19,8 +19,6 @@ static void *pgBackupValidateFiles(void *arg); static void do_validate_instance(void); static bool corrupted_backup_found = false; -/* Program version of a current backup */ -static uint32 validate_backup_version = 0; typedef struct { @@ -220,7 +218,7 @@ pgBackupValidateFiles(void *arg) * To avoid this problem we need to use different algorithm, CRC-32 in * this case. */ - crc = pgFileGetCRC(file->path, validate_backup_version <= 20021); + crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", diff --git a/src/validate.c.autosave b/src/validate.c.autosave new file mode 100644 index 00000000..1c07e0ae --- /dev/null +++ b/src/validate.c.autosave @@ -0,0 +1,532 @@ +/*------------------------------------------------------------------------- + * + * validate.c: validate backup files. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); +static void do_validate_instance(void); + +static bool corrupted_backup_found = false; + +typedef struct +{ + parray *files; + bool corrupted; + XLogRecPtr stop_lsn; + uint32 checksum_version; + uint32 backup_version; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} validate_files_arg; + +if (is_absolute_path(host)) + if (hostaddr && *hostaddr) + printf(_("You are connected to database \"%s\" as user \"%s\" on address \"%s\" at port \"%s\".\n"), + db, PQuser(pset.db), hostaddr, PQport(pset.db)); + else + printf(_("You are connected to database \"%s\" as user \"%s\" via socket in \"%s\" at port \"%s\".\n"), + db, PQuser(pset.db), host, PQport(pset.db)); +else + if (hostaddr && *hostaddr && strcmp(host, hostaddr) != 0) + printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" (address \"%s\") at port \"%s\".\n"), + db, PQuser(pset.db), host, hostaddr, PQport(pset.db)); + else + printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" at port \"%s\".\n"), + db, PQuser(pset.db), host, PQport(pset.db)); + +/* + * Validate backup files. + */ +void +pgBackupValidate(pgBackup *backup) +{ + char base_path[MAXPGPATH]; + char path[MAXPGPATH]; + parray *files; + bool corrupted = false; + bool validation_isok = true; + /* arrays with meta info for multi threaded validate */ + pthread_t *threads; + validate_files_arg *threads_args; + int i; + + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE && + backup->status != BACKUP_STATUS_ORPHAN && + backup->status != BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s has status %s. Skip validation.", + base36enc(backup->start_time), status2str(backup->status)); + corrupted_backup_found = true; + return; + } + + if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) + elog(INFO, "Validating backup %s", base36enc(backup->start_time)); + /* backups in MERGING status must have an option of revalidation without losing MERGING status + else if (backup->status == BACKUP_STATUS_MERGING) + { + some message here + } + */ + else + elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); + + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->backup_mode != BACKUP_MODE_DIFF_PAGE && + backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && + backup->backup_mode != BACKUP_MODE_DIFF_DELTA) + elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); + + pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + files = dir_read_file_list(base_path, path); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + pg_atomic_clear_flag(&file->lock); + } + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (validate_files_arg *) + palloc(sizeof(validate_files_arg) * num_threads); + + /* Validate files */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->corrupted = false; + arg->stop_lsn = backup->stop_lsn; + arg->checksum_version = backup->checksum_version; + arg->backup_version = parse_program_version(backup->program_version); + /* By default there are some error */ + threads_args[i].ret = 1; + + pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + pthread_join(threads[i], NULL); + if (arg->corrupted) + corrupted = true; + if (arg->ret == 1) + validation_isok = false; + } + if (!validation_isok) + elog(ERROR, "Data files validation failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + /* Update backup status */ + backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; + write_backup_status(backup); + + if (corrupted) + elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); + else + elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); +} + +/* + * Validate files in the backup. + * NOTE: If file is not valid, do not use ERROR log message, + * rather throw a WARNING and set arguments->corrupted = true. + * This is necessary to update backup status. + */ +static void * +pgBackupValidateFiles(void *arg) +{ + int i; + validate_files_arg *arguments = (validate_files_arg *)arg; + pg_crc32 crc; + + for (i = 0; i < parray_num(arguments->files); i++) + { + struct stat st; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + if (interrupted) + elog(ERROR, "Interrupted during validate"); + + /* Validate only regular files */ + if (!S_ISREG(file->mode)) + continue; + /* + * Skip files which has no data, because they + * haven't changed between backups. + */ + if (file->write_size == BYTES_INVALID) + continue; + + /* + * Currently we don't compute checksums for + * cfs_compressed data files, so skip them. + */ + if (file->is_cfs) + continue; + + /* print progress */ + elog(VERBOSE, "Validate files: (%d/%lu) %s", + i + 1, (unsigned long) parray_num(arguments->files), file->path); + + if (stat(file->path, &st) == -1) + { + if (errno == ENOENT) + elog(WARNING, "Backup file \"%s\" is not found", file->path); + else + elog(WARNING, "Cannot stat backup file \"%s\": %s", + file->path, strerror(errno)); + arguments->corrupted = true; + break; + } + + if (file->write_size != st.st_size) + { + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); + arguments->corrupted = true; + break; + } + + /* + * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we + * use CRC-32. + * + * pg_control stores its content and checksum of the content, calculated + * using CRC-32C. If we calculate checksum of the whole pg_control using + * CRC-32C we get same checksum constantly. It might be because of the + * CRC-32C algorithm. + * To avoid this problem we need to use different algorithm, CRC-32 in + * this case. + */ + crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + arguments->corrupted = true; + + /* validate relation blocks */ + if (file->is_datafile) + { + if (!check_file_pages(file, arguments->stop_lsn, + arguments->checksum_version, + arguments->backup_version)) + arguments->corrupted = true; + } + } + } + + /* Data files validation is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Validate all backups in the backup catalog. + * If --instance option was provided, validate only backups of this instance. + */ +int +do_validate_all(void) +{ + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + xlog_seg_size = get_config_xlog_seg_size(); + + do_validate_instance(); + } + } + else + { + do_validate_instance(); + } + + if (corrupted_backup_found) + { + elog(WARNING, "Some backups are not valid"); + return 1; + } + else + elog(INFO, "All backups are valid"); + + return 0; +} + +/* + * Validate all backups in the given instance of the backup catalog. + */ +static void +do_validate_instance(void) +{ + char *current_backup_id; + int i; + int j; + parray *backups; + pgBackup *current_backup = NULL; + + elog(INFO, "Validate backups of the instance '%s'", instance_name); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Examine backups one by one and validate them */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *base_full_backup; + char *parent_backup_id; + + current_backup = (pgBackup *) parray_get(backups, i); + + /* Find ancestor for incremental backup */ + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + pgBackup *tmp_backup = NULL; + int result; + + result = scan_parent_chain(current_backup, &tmp_backup); + + /* chain is broken */ + if (result == 0) + { + /* determine missing backup ID */ + + parent_backup_id = base36enc_dup(tmp_backup->parent_backup); + corrupted_backup_found = true; + + /* orphanize current_backup */ + if (current_backup->status == BACKUP_STATUS_OK) + { + current_backup->status = BACKUP_STATUS_ORPHAN; + write_backup_status(current_backup); + elog(WARNING, "Backup %s is orphaned because his parent %s is missing", + base36enc(current_backup->start_time), + parent_backup_id); + } + else + { + elog(WARNING, "Backup %s has missing parent %s", + base36enc(current_backup->start_time), parent_backup_id); + } + continue; + } + /* chain is whole, but at least one parent is invalid */ + else if (result == 1) + { + /* determine corrupt backup ID */ + parent_backup_id = base36enc_dup(tmp_backup->start_time); + + /* Oldest corrupt backup has a chance for revalidation */ + if (current_backup->start_time != tmp_backup->start_time) + { + /* orphanize current_backup */ + if (current_backup->status == BACKUP_STATUS_OK) + { + current_backup->status = BACKUP_STATUS_ORPHAN; + write_backup_status(current_backup); + elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", + base36enc(current_backup->start_time), parent_backup_id, + status2str(tmp_backup->status)); + } + else + { + elog(WARNING, "Backup %s has parent %s with status: %s", + base36enc(current_backup->start_time),parent_backup_id, + status2str(tmp_backup->status)); + } + continue; + } + base_full_backup = find_parent_full_backup(current_backup); + } + /* chain is whole, all parents are valid at first glance, + * current backup validation can proceed + */ + else + base_full_backup = tmp_backup; + } + else + base_full_backup = current_backup; + + /* Valiate backup files*/ + pgBackupValidate(current_backup); + + /* Validate corresponding WAL files */ + if (current_backup->status == BACKUP_STATUS_OK) + validate_wal(current_backup, arclog_path, 0, + 0, 0, base_full_backup->tli, xlog_seg_size); + + /* + * Mark every descendant of corrupted backup as orphan + */ + if (current_backup->status == BACKUP_STATUS_CORRUPT) + { + /* This is ridiculous but legal. + * PAGE1_2b <- OK + * PAGE1_2a <- OK + * PAGE1_1b <- ORPHAN + * PAGE1_1a <- CORRUPT + * FULL1 <- OK + */ + + corrupted_backup_found = true; + current_backup_id = base36enc_dup(current_backup->start_time); + + for (j = i - 1; j >= 0; j--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, j); + + if (is_parent(current_backup->start_time, backup, false)) + { + if (backup->status == BACKUP_STATUS_OK) + { + backup->status = BACKUP_STATUS_ORPHAN; + write_backup_status(backup); + + elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", + base36enc(backup->start_time), + current_backup_id, + status2str(current_backup->status)); + } + } + } + free(current_backup_id); + } + + /* For every OK backup we try to revalidate all his ORPHAN descendants. */ + if (current_backup->status == BACKUP_STATUS_OK) + { + /* revalidate all ORPHAN descendats + * be very careful not to miss a missing backup + * for every backup we must check that he is descendant of current_backup + */ + for (j = i - 1; j >= 0; j--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, j); + pgBackup *tmp_backup = NULL; + int result; + + //PAGE3b ORPHAN + //PAGE2b ORPHAN ----- + //PAGE6a ORPHAN | + //PAGE5a CORRUPT | + //PAGE4a missing | + //PAGE3a missing | + //PAGE2a ORPHAN | + //PAGE1a OK <- we are here <-| + //FULL OK + + if (is_parent(current_backup->start_time, backup, false)) + { + /* Revalidation make sense only if parent chain is whole. + * is_parent() do not guarantee that. + */ + result = scan_parent_chain(backup, &tmp_backup); + + if (result == 1) + { + /* revalidation make sense only if oldest invalid backup is current_backup + */ + + if (tmp_backup->start_time != backup->start_time) + continue; + + if (backup->status == BACKUP_STATUS_ORPHAN) + { + /* Revaliate backup files*/ + pgBackupValidate(backup); + + if (backup->status == BACKUP_STATUS_OK) + { + //tmp_backup = find_parent_full_backup(dest_backup); + /* Revalidation successful, validate corresponding WAL files */ + validate_wal(backup, arclog_path, 0, + 0, 0, current_backup->tli, + xlog_seg_size); + } + } + + if (backup->status != BACKUP_STATUS_OK) + { + corrupted_backup_found = true; + continue; + } + } + } + } + } + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); +} From cf6e7445e5a39876da304fdab38c13a682f11d5c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 8 Nov 2018 15:51:58 +0300 Subject: [PATCH 15/37] Remove .autosave --- src/validate.c.autosave | 532 ---------------------------------------- 1 file changed, 532 deletions(-) delete mode 100644 src/validate.c.autosave diff --git a/src/validate.c.autosave b/src/validate.c.autosave deleted file mode 100644 index 1c07e0ae..00000000 --- a/src/validate.c.autosave +++ /dev/null @@ -1,532 +0,0 @@ -/*------------------------------------------------------------------------- - * - * validate.c: validate backup files. - * - * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include - -#include "utils/thread.h" - -static void *pgBackupValidateFiles(void *arg); -static void do_validate_instance(void); - -static bool corrupted_backup_found = false; - -typedef struct -{ - parray *files; - bool corrupted; - XLogRecPtr stop_lsn; - uint32 checksum_version; - uint32 backup_version; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} validate_files_arg; - -if (is_absolute_path(host)) - if (hostaddr && *hostaddr) - printf(_("You are connected to database \"%s\" as user \"%s\" on address \"%s\" at port \"%s\".\n"), - db, PQuser(pset.db), hostaddr, PQport(pset.db)); - else - printf(_("You are connected to database \"%s\" as user \"%s\" via socket in \"%s\" at port \"%s\".\n"), - db, PQuser(pset.db), host, PQport(pset.db)); -else - if (hostaddr && *hostaddr && strcmp(host, hostaddr) != 0) - printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" (address \"%s\") at port \"%s\".\n"), - db, PQuser(pset.db), host, hostaddr, PQport(pset.db)); - else - printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" at port \"%s\".\n"), - db, PQuser(pset.db), host, PQport(pset.db)); - -/* - * Validate backup files. - */ -void -pgBackupValidate(pgBackup *backup) -{ - char base_path[MAXPGPATH]; - char path[MAXPGPATH]; - parray *files; - bool corrupted = false; - bool validation_isok = true; - /* arrays with meta info for multi threaded validate */ - pthread_t *threads; - validate_files_arg *threads_args; - int i; - - /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE && - backup->status != BACKUP_STATUS_ORPHAN && - backup->status != BACKUP_STATUS_CORRUPT) - { - elog(WARNING, "Backup %s has status %s. Skip validation.", - base36enc(backup->start_time), status2str(backup->status)); - corrupted_backup_found = true; - return; - } - - if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) - elog(INFO, "Validating backup %s", base36enc(backup->start_time)); - /* backups in MERGING status must have an option of revalidation without losing MERGING status - else if (backup->status == BACKUP_STATUS_MERGING) - { - some message here - } - */ - else - elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); - - if (backup->backup_mode != BACKUP_MODE_FULL && - backup->backup_mode != BACKUP_MODE_DIFF_PAGE && - backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && - backup->backup_mode != BACKUP_MODE_DIFF_DELTA) - elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); - - pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); - pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - files = dir_read_file_list(base_path, path); - - /* setup threads */ - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - pg_atomic_clear_flag(&file->lock); - } - - /* init thread args with own file lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (validate_files_arg *) - palloc(sizeof(validate_files_arg) * num_threads); - - /* Validate files */ - for (i = 0; i < num_threads; i++) - { - validate_files_arg *arg = &(threads_args[i]); - - arg->files = files; - arg->corrupted = false; - arg->stop_lsn = backup->stop_lsn; - arg->checksum_version = backup->checksum_version; - arg->backup_version = parse_program_version(backup->program_version); - /* By default there are some error */ - threads_args[i].ret = 1; - - pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); - } - - /* Wait theads */ - for (i = 0; i < num_threads; i++) - { - validate_files_arg *arg = &(threads_args[i]); - - pthread_join(threads[i], NULL); - if (arg->corrupted) - corrupted = true; - if (arg->ret == 1) - validation_isok = false; - } - if (!validation_isok) - elog(ERROR, "Data files validation failed"); - - pfree(threads); - pfree(threads_args); - - /* cleanup */ - parray_walk(files, pgFileFree); - parray_free(files); - - /* Update backup status */ - backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; - write_backup_status(backup); - - if (corrupted) - elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); - else - elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); -} - -/* - * Validate files in the backup. - * NOTE: If file is not valid, do not use ERROR log message, - * rather throw a WARNING and set arguments->corrupted = true. - * This is necessary to update backup status. - */ -static void * -pgBackupValidateFiles(void *arg) -{ - int i; - validate_files_arg *arguments = (validate_files_arg *)arg; - pg_crc32 crc; - - for (i = 0; i < parray_num(arguments->files); i++) - { - struct stat st; - pgFile *file = (pgFile *) parray_get(arguments->files, i); - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - if (interrupted) - elog(ERROR, "Interrupted during validate"); - - /* Validate only regular files */ - if (!S_ISREG(file->mode)) - continue; - /* - * Skip files which has no data, because they - * haven't changed between backups. - */ - if (file->write_size == BYTES_INVALID) - continue; - - /* - * Currently we don't compute checksums for - * cfs_compressed data files, so skip them. - */ - if (file->is_cfs) - continue; - - /* print progress */ - elog(VERBOSE, "Validate files: (%d/%lu) %s", - i + 1, (unsigned long) parray_num(arguments->files), file->path); - - if (stat(file->path, &st) == -1) - { - if (errno == ENOENT) - elog(WARNING, "Backup file \"%s\" is not found", file->path); - else - elog(WARNING, "Cannot stat backup file \"%s\": %s", - file->path, strerror(errno)); - arguments->corrupted = true; - break; - } - - if (file->write_size != st.st_size) - { - elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", - file->path, file->write_size, (unsigned long) st.st_size); - arguments->corrupted = true; - break; - } - - /* - * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we - * use CRC-32. - * - * pg_control stores its content and checksum of the content, calculated - * using CRC-32C. If we calculate checksum of the whole pg_control using - * CRC-32C we get same checksum constantly. It might be because of the - * CRC-32C algorithm. - * To avoid this problem we need to use different algorithm, CRC-32 in - * this case. - */ - crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); - if (crc != file->crc) - { - elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", - file->path, file->crc, crc); - arguments->corrupted = true; - - /* validate relation blocks */ - if (file->is_datafile) - { - if (!check_file_pages(file, arguments->stop_lsn, - arguments->checksum_version, - arguments->backup_version)) - arguments->corrupted = true; - } - } - } - - /* Data files validation is successful */ - arguments->ret = 0; - - return NULL; -} - -/* - * Validate all backups in the backup catalog. - * If --instance option was provided, validate only backups of this instance. - */ -int -do_validate_all(void) -{ - if (instance_name == NULL) - { - /* Show list of instances */ - char path[MAXPGPATH]; - DIR *dir; - struct dirent *dent; - - /* open directory and list contents */ - join_path_components(path, backup_path, BACKUPS_DIR); - dir = opendir(path); - if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); - - errno = 0; - while ((dent = readdir(dir))) - { - char child[MAXPGPATH]; - struct stat st; - - /* skip entries point current dir or parent dir */ - if (strcmp(dent->d_name, ".") == 0 || - strcmp(dent->d_name, "..") == 0) - continue; - - join_path_components(child, path, dent->d_name); - - if (lstat(child, &st) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); - - if (!S_ISDIR(st.st_mode)) - continue; - - instance_name = dent->d_name; - sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); - xlog_seg_size = get_config_xlog_seg_size(); - - do_validate_instance(); - } - } - else - { - do_validate_instance(); - } - - if (corrupted_backup_found) - { - elog(WARNING, "Some backups are not valid"); - return 1; - } - else - elog(INFO, "All backups are valid"); - - return 0; -} - -/* - * Validate all backups in the given instance of the backup catalog. - */ -static void -do_validate_instance(void) -{ - char *current_backup_id; - int i; - int j; - parray *backups; - pgBackup *current_backup = NULL; - - elog(INFO, "Validate backups of the instance '%s'", instance_name); - - /* Get exclusive lock of backup catalog */ - catalog_lock(); - - /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(INVALID_BACKUP_ID); - - /* Examine backups one by one and validate them */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *base_full_backup; - char *parent_backup_id; - - current_backup = (pgBackup *) parray_get(backups, i); - - /* Find ancestor for incremental backup */ - if (current_backup->backup_mode != BACKUP_MODE_FULL) - { - pgBackup *tmp_backup = NULL; - int result; - - result = scan_parent_chain(current_backup, &tmp_backup); - - /* chain is broken */ - if (result == 0) - { - /* determine missing backup ID */ - - parent_backup_id = base36enc_dup(tmp_backup->parent_backup); - corrupted_backup_found = true; - - /* orphanize current_backup */ - if (current_backup->status == BACKUP_STATUS_OK) - { - current_backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(current_backup); - elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - base36enc(current_backup->start_time), - parent_backup_id); - } - else - { - elog(WARNING, "Backup %s has missing parent %s", - base36enc(current_backup->start_time), parent_backup_id); - } - continue; - } - /* chain is whole, but at least one parent is invalid */ - else if (result == 1) - { - /* determine corrupt backup ID */ - parent_backup_id = base36enc_dup(tmp_backup->start_time); - - /* Oldest corrupt backup has a chance for revalidation */ - if (current_backup->start_time != tmp_backup->start_time) - { - /* orphanize current_backup */ - if (current_backup->status == BACKUP_STATUS_OK) - { - current_backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(current_backup); - elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(current_backup->start_time), parent_backup_id, - status2str(tmp_backup->status)); - } - else - { - elog(WARNING, "Backup %s has parent %s with status: %s", - base36enc(current_backup->start_time),parent_backup_id, - status2str(tmp_backup->status)); - } - continue; - } - base_full_backup = find_parent_full_backup(current_backup); - } - /* chain is whole, all parents are valid at first glance, - * current backup validation can proceed - */ - else - base_full_backup = tmp_backup; - } - else - base_full_backup = current_backup; - - /* Valiate backup files*/ - pgBackupValidate(current_backup); - - /* Validate corresponding WAL files */ - if (current_backup->status == BACKUP_STATUS_OK) - validate_wal(current_backup, arclog_path, 0, - 0, 0, base_full_backup->tli, xlog_seg_size); - - /* - * Mark every descendant of corrupted backup as orphan - */ - if (current_backup->status == BACKUP_STATUS_CORRUPT) - { - /* This is ridiculous but legal. - * PAGE1_2b <- OK - * PAGE1_2a <- OK - * PAGE1_1b <- ORPHAN - * PAGE1_1a <- CORRUPT - * FULL1 <- OK - */ - - corrupted_backup_found = true; - current_backup_id = base36enc_dup(current_backup->start_time); - - for (j = i - 1; j >= 0; j--) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - if (is_parent(current_backup->start_time, backup, false)) - { - if (backup->status == BACKUP_STATUS_OK) - { - backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(backup); - - elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(backup->start_time), - current_backup_id, - status2str(current_backup->status)); - } - } - } - free(current_backup_id); - } - - /* For every OK backup we try to revalidate all his ORPHAN descendants. */ - if (current_backup->status == BACKUP_STATUS_OK) - { - /* revalidate all ORPHAN descendats - * be very careful not to miss a missing backup - * for every backup we must check that he is descendant of current_backup - */ - for (j = i - 1; j >= 0; j--) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - pgBackup *tmp_backup = NULL; - int result; - - //PAGE3b ORPHAN - //PAGE2b ORPHAN ----- - //PAGE6a ORPHAN | - //PAGE5a CORRUPT | - //PAGE4a missing | - //PAGE3a missing | - //PAGE2a ORPHAN | - //PAGE1a OK <- we are here <-| - //FULL OK - - if (is_parent(current_backup->start_time, backup, false)) - { - /* Revalidation make sense only if parent chain is whole. - * is_parent() do not guarantee that. - */ - result = scan_parent_chain(backup, &tmp_backup); - - if (result == 1) - { - /* revalidation make sense only if oldest invalid backup is current_backup - */ - - if (tmp_backup->start_time != backup->start_time) - continue; - - if (backup->status == BACKUP_STATUS_ORPHAN) - { - /* Revaliate backup files*/ - pgBackupValidate(backup); - - if (backup->status == BACKUP_STATUS_OK) - { - //tmp_backup = find_parent_full_backup(dest_backup); - /* Revalidation successful, validate corresponding WAL files */ - validate_wal(backup, arclog_path, 0, - 0, 0, current_backup->tli, - xlog_seg_size); - } - } - - if (backup->status != BACKUP_STATUS_OK) - { - corrupted_backup_found = true; - continue; - } - } - } - } - } - } - - /* cleanup */ - parray_walk(backups, pgBackupFree); - parray_free(backups); -} From a0ec849b4a0fdf6a746a8e923fe566d1abca0a02 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 8 Nov 2018 18:13:42 +0300 Subject: [PATCH 16/37] We should check only negative return values --- src/data.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index d3e48e9a..eeb2952d 100644 --- a/src/data.c +++ b/src/data.c @@ -69,7 +69,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, { int32 ret; ret = zlib_compress(dst, dst_size, src, src_size, level); - if (ret != Z_OK && errormsg) + if (ret < Z_OK && errormsg) *errormsg = zError(ret); return ret; } @@ -99,7 +99,7 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, { int32 ret; ret = zlib_decompress(dst, dst_size, src, src_size); - if (ret != Z_OK && errormsg) + if (ret < Z_OK && errormsg) *errormsg = zError(ret); return ret; } From c530d28869d9865fd98d13c0b723db5f3d88779c Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 8 Nov 2018 19:38:22 +0300 Subject: [PATCH 17/37] change datafile validation algorithm: - validate file block by block by default, not only in case of file-level checksum corruption; - add an option: --skip-block-validation to disable this behaviour; - calculate file checksum at the same time as validate blocks; --- src/data.c | 22 ++++++++++++++++++-- src/dir.c | 30 ++++---------------------- src/help.c | 11 +++++++++- src/pg_probackup.c | 3 +++ src/pg_probackup.h | 27 ++++++++++++++++++++++-- src/validate.c | 52 +++++++++++++++++++++++++++------------------- 6 files changed, 93 insertions(+), 52 deletions(-) diff --git a/src/data.c b/src/data.c index eeb2952d..4fc53783 100644 --- a/src/data.c +++ b/src/data.c @@ -1601,12 +1601,14 @@ validate_one_page(Page page, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool -check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, - uint32 backup_version) +check_file_pages(pgFile *file, XLogRecPtr stop_lsn, + uint32 checksum_version, uint32 backup_version) { size_t read_len = 0; bool is_valid = true; FILE *in; + pg_crc32 crc; + bool use_crc32c = (backup_version <= 20021); elog(VERBOSE, "validate relation blocks for file %s", file->name); @@ -1623,6 +1625,9 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, file->path, strerror(errno)); } + /* calc CRC of backup file */ + INIT_FILE_CRC32(use_crc32c, crc); + /* read and validate pages one by one */ while (true) { @@ -1647,6 +1652,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, blknum, file->path, strerror(errno_tmp)); } + COMP_FILE_CRC32(use_crc32c, crc, &header, read_len); + if (header.block < blknum) elog(ERROR, "backup is broken at file->path %s block %u", file->path, blknum); @@ -1668,6 +1675,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", blknum, file->path, read_len, header.compressed_size); + COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); + if (header.compressed_size != BLCKSZ || page_may_be_compressed(compressed_page.data, file->compress_alg, backup_version)) @@ -1706,5 +1715,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, } } + FIN_FILE_CRC32(use_crc32c, crc); + + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + is_valid = false; + } + return is_valid; } diff --git a/src/dir.c b/src/dir.c index 9e55c821..5c6f60d8 100644 --- a/src/dir.c +++ b/src/dir.c @@ -267,28 +267,6 @@ pgFileGetCRC(const char *file_path, bool use_crc32c) size_t len; int errno_tmp; -#define INIT_FILE_CRC32(crc) \ -do { \ - if (use_crc32c) \ - INIT_CRC32C(crc); \ - else \ - INIT_TRADITIONAL_CRC32(crc); \ -} while (0) -#define COMP_FILE_CRC32(crc, data, len) \ -do { \ - if (use_crc32c) \ - COMP_CRC32C((crc), (data), (len)); \ - else \ - COMP_TRADITIONAL_CRC32(crc, data, len); \ -} while (0) -#define FIN_FILE_CRC32(crc) \ -do { \ - if (use_crc32c) \ - FIN_CRC32C(crc); \ - else \ - FIN_TRADITIONAL_CRC32(crc); \ -} while (0) - /* open file in binary read mode */ fp = fopen(file_path, PG_BINARY_R); if (fp == NULL) @@ -296,20 +274,20 @@ do { \ file_path, strerror(errno)); /* calc CRC of backup file */ - INIT_FILE_CRC32(crc); + INIT_FILE_CRC32(use_crc32c, crc); while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf)) { if (interrupted) elog(ERROR, "interrupted during CRC calculation"); - COMP_FILE_CRC32(crc, buf, len); + COMP_FILE_CRC32(use_crc32c, crc, buf, len); } errno_tmp = errno; if (!feof(fp)) elog(WARNING, "cannot read \"%s\": %s", file_path, strerror(errno_tmp)); if (len > 0) - COMP_FILE_CRC32(crc, buf, len); - FIN_FILE_CRC32(crc); + COMP_FILE_CRC32(use_crc32c, crc, buf, len); + FIN_FILE_CRC32(use_crc32c, crc); fclose(fp); diff --git a/src/help.c b/src/help.c index 1cf6d404..409c8f82 100644 --- a/src/help.c +++ b/src/help.c @@ -118,6 +118,7 @@ help_pg_probackup(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--skip-block-validation]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); @@ -127,12 +128,14 @@ help_pg_probackup(void) printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); + printf(_(" [--skip-block-validation]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--timeline=timeline]\n")); + printf(_(" [--skip-block-validation]\n")); printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); @@ -203,7 +206,8 @@ help_backup(void) printf(_(" [-w --no-password] [-W --password]\n")); printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--skip-block-validation]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -215,6 +219,7 @@ help_backup(void) printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_(" --progress show progress\n")); + printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -279,6 +284,7 @@ help_restore(void) printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica] [--no-validate]\n\n")); + printf(_(" [--skip-block-validation]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -305,6 +311,7 @@ help_restore(void) printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); printf(_(" to ease setting up a standby server\n")); printf(_(" --no-validate disable backup validation during restore\n")); + printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -335,6 +342,7 @@ help_validate(void) printf(_(" [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline]\n\n")); + printf(_(" [--skip-block-validation]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -348,6 +356,7 @@ help_validate(void) printf(_(" --timeline=timeline recovering into a particular timeline\n")); printf(_(" --recovery-target-name=target-name\n")); printf(_(" the named restore point to which recovery will proceed\n")); + printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 82107097..ea99672d 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -89,6 +89,8 @@ static pgRecoveryTarget *recovery_target_options = NULL; bool restore_as_replica = false; bool restore_no_validate = false; +bool skip_block_validation = false; + /* delete options */ bool delete_wal = false; bool delete_expired = false; @@ -179,6 +181,7 @@ static pgut_option options[] = { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, + { 'b', 29, "skip-block-validation", &skip_block_validation, SOURCE_CMDLINE }, /* delete options */ { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 182a647b..c6ff6343 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -65,6 +65,28 @@ typedef enum CompressAlg ZLIB_COMPRESS, } CompressAlg; +#define INIT_FILE_CRC32(use_crc32c, crc) \ +do { \ + if (use_crc32c) \ + INIT_CRC32C(crc); \ + else \ + INIT_TRADITIONAL_CRC32(crc); \ +} while (0) +#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \ +do { \ + if (use_crc32c) \ + COMP_CRC32C((crc), (data), (len)); \ + else \ + COMP_TRADITIONAL_CRC32(crc, data, len); \ +} while (0) +#define FIN_FILE_CRC32(use_crc32c, crc) \ +do { \ + if (use_crc32c) \ + FIN_CRC32C(crc); \ + else \ + FIN_TRADITIONAL_CRC32(crc); \ +} while (0) + /* Information about single file (or dir) in backup */ typedef struct pgFile { @@ -339,6 +361,7 @@ extern bool exclusive_backup; /* restore options */ extern bool restore_as_replica; +extern bool skip_block_validation; /* delete options */ extern bool delete_wal; @@ -527,9 +550,9 @@ extern void get_wal_file(const char *from_path, const char *to_path); extern bool calc_file_checksum(pgFile *file); -extern bool check_file_pages(pgFile* file, XLogRecPtr stop_lsn, +extern bool check_file_pages(pgFile* file, + XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); - /* parsexlog.c */ extern void extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, diff --git a/src/validate.c b/src/validate.c index 2f75cd4b..55ea5ebe 100644 --- a/src/validate.c +++ b/src/validate.c @@ -208,32 +208,42 @@ pgBackupValidateFiles(void *arg) } /* - * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we - * use CRC-32. - * - * pg_control stores its content and checksum of the content, calculated - * using CRC-32C. If we calculate checksum of the whole pg_control using - * CRC-32C we get same checksum constantly. It might be because of the - * CRC-32C algorithm. - * To avoid this problem we need to use different algorithm, CRC-32 in - * this case. + * If option skip-block-validation is set, compute only file-level CRC for + * datafiles, otherwise check them block by block. */ - crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); - if (crc != file->crc) + if (!file->is_datafile || skip_block_validation) { - elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", - file->path, file->crc, crc); - arguments->corrupted = true; - - /* validate relation blocks */ - if (file->is_datafile) + /* + * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we + * use CRC-32. + * + * pg_control stores its content and checksum of the content, calculated + * using CRC-32C. If we calculate checksum of the whole pg_control using + * CRC-32C we get same checksum constantly. It might be because of the + * CRC-32C algorithm. + * To avoid this problem we need to use different algorithm, CRC-32 in + * this case. + */ + crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); + if (crc != file->crc) { - if (!check_file_pages(file, arguments->stop_lsn, - arguments->checksum_version, - arguments->backup_version)) - arguments->corrupted = true; + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + arguments->corrupted = true; } } + else + { + /* + * validate relation block by block + * check page headers, checksums (if enabled) + * and compute checksum of the file + */ + if (!check_file_pages(file, arguments->stop_lsn, + arguments->checksum_version, + arguments->backup_version)) + arguments->corrupted = true; + } } /* Data files validation is successful */ From 141ba33000d21fc3b5f78e9678552b6acf28f40d Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 8 Nov 2018 20:49:30 +0300 Subject: [PATCH 18/37] fix: close file after validation --- src/data.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/data.c b/src/data.c index 4fc53783..1d1a2ee9 100644 --- a/src/data.c +++ b/src/data.c @@ -1716,6 +1716,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, } FIN_FILE_CRC32(use_crc32c, crc); + fclose(in); if (crc != file->crc) { From 4870fa7f68728cb51bc81f3d4f74cd53b1dd76f6 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 9 Nov 2018 13:59:56 +0300 Subject: [PATCH 19/37] more informative error message for failed WAL record read --- src/parsexlog.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 7f7365f5..ee7b5076 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -237,10 +237,11 @@ doExtractPageMap(void *arg) */ if (XLogRecPtrIsInvalid(found)) { - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X. %s", private_data->thread_num, (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint)); + (uint32) (extract_arg->startpoint), + (xlogreader->errormsg_buf[0] != '\0')?xlogreader->errormsg_buf:""); PrintXLogCorruptionMsg(private_data, ERROR); } extract_arg->startpoint = found; From 12a6dcdd93ec5dddc2d24df326005a91c41b36d5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 9 Nov 2018 18:45:29 +0300 Subject: [PATCH 20/37] PGPRO-1905: check message about system id mismatch --- tests/page.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/page.py b/tests/page.py index 3d19a81d..1d8238e1 100644 --- a/tests/page.py +++ b/tests/page.py @@ -936,6 +936,8 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and 'could not read WAL record at' in e.message and + 'WAL file is from different database system: WAL file database system identifier is' in e.message and + 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file_destination) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -961,6 +963,8 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and 'could not read WAL record at' in e.message and + 'WAL file is from different database system: WAL file database system identifier is' in e.message and + 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file_destination) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( From 4995652ef5b1a225e6efb90b9512704654078009 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Sat, 10 Nov 2018 15:49:36 +0300 Subject: [PATCH 21/37] fix decompression of BLCKSZ pages --- src/data.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/data.c b/src/data.c index 1d1a2ee9..d9fd8837 100644 --- a/src/data.c +++ b/src/data.c @@ -722,6 +722,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, size_t read_len; DataPage compressed_page; /* used as read buffer */ DataPage page; + int32 uncompressed_size = 0; /* File didn`t changed. Nothig to copy */ if (file->write_size == BYTES_INVALID) @@ -777,17 +778,23 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, Assert(header.compressed_size <= BLCKSZ); + /* read a page from file */ read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", blknum, file->path, read_len, header.compressed_size); + /* + * if page size is smaller than BLCKSZ, decompress the page. + * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. + * we have to check, whether it is compressed or not using + * page_may_be_compressed() function. + */ if (header.compressed_size != BLCKSZ || page_may_be_compressed(compressed_page.data, file->compress_alg, backup_version)) { - int32 uncompressed_size = 0; const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, @@ -820,7 +827,11 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, strerror(errno)); } - if (header.compressed_size < BLCKSZ) + /* if we uncompressed the page - write page.data, + * if page wasn't compressed - + * write what we've read - compressed_page.data + */ + if (uncompressed_size == BLCKSZ) { if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", @@ -828,7 +839,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, } else { - /* if page wasn't compressed, we've read full block */ + /* */ if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", blknum, file->path, strerror(errno)); From 88354a8dde65b0b60f1f64450b84596303b12b23 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 16 Oct 2018 18:13:27 +0300 Subject: [PATCH 22/37] PGPRO-1892: Continue failed merge command --- src/backup.c | 2 +- src/catalog.c | 14 ++++-- src/data.c | 16 ------- src/delete.c | 27 ++++++----- src/merge.c | 114 ++++++++++++++++++++------------------------- src/pg_probackup.h | 7 +-- tests/merge.py | 9 +++- 7 files changed, 88 insertions(+), 101 deletions(-) diff --git a/src/backup.c b/src/backup.c index b2aed318..036f3e91 100644 --- a/src/backup.c +++ b/src/backup.c @@ -790,7 +790,7 @@ do_backup_instance(void) } /* Print the list of files to backup catalog */ - pgBackupWriteFileList(¤t, backup_files_list, pgdata); + write_backup_filelist(¤t, backup_files_list, pgdata); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) diff --git a/src/catalog.c b/src/catalog.c index 41676f4c..788c33f7 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -509,18 +509,22 @@ write_backup(pgBackup *backup) fp = fopen(conf_path, "wt"); if (fp == NULL) elog(ERROR, "Cannot open configuration file \"%s\": %s", conf_path, - strerror(errno)); + strerror(errno)); pgBackupWriteControl(fp, backup); - fclose(fp); + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "Cannot write configuration file \"%s\": %s", + conf_path, strerror(errno)); } /* * Output the list of files to backup catalog DATABASE_FILE_LIST */ void -pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) +write_backup_filelist(pgBackup *backup, parray *files, const char *root) { FILE *fp; char path[MAXPGPATH]; @@ -529,7 +533,7 @@ pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) fp = fopen(path, "wt"); if (fp == NULL) - elog(ERROR, "cannot open file list \"%s\": %s", path, + elog(ERROR, "Cannot open file list \"%s\": %s", path, strerror(errno)); print_file_list(fp, files, root); @@ -537,7 +541,7 @@ pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || fclose(fp)) - elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot write file list \"%s\": %s", path, strerror(errno)); } /* diff --git a/src/data.c b/src/data.c index d9fd8837..9b6cbd23 100644 --- a/src/data.c +++ b/src/data.c @@ -1037,22 +1037,6 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) return true; } -/* - * Move file from one backup to another. - * We do not apply compression to these files, because - * it is either small control file or already compressed cfs file. - */ -void -move_file(const char *from_root, const char *to_root, pgFile *file) -{ - char to_path[MAXPGPATH]; - - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - if (rename(file->path, to_path) == -1) - elog(ERROR, "Cannot move file \"%s\" to path \"%s\": %s", - file->path, to_path, strerror(errno)); -} - #ifdef HAVE_LIBZ /* * Show error during work with compressed file diff --git a/src/delete.c b/src/delete.c index c5f16af7..599fd2fd 100644 --- a/src/delete.c +++ b/src/delete.c @@ -14,7 +14,6 @@ #include #include -static int delete_backup_files(pgBackup *backup); static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, uint32 xlog_seg_size); @@ -245,7 +244,7 @@ do_retention_purge(void) * Delete backup files of the backup and update the status of the backup to * BACKUP_STATUS_DELETED. */ -static int +void delete_backup_files(pgBackup *backup) { size_t i; @@ -257,11 +256,15 @@ delete_backup_files(pgBackup *backup) * If the backup was deleted already, there is nothing to do. */ if (backup->status == BACKUP_STATUS_DELETED) - return 0; + { + elog(WARNING, "Backup %s already deleted", + base36enc(backup->start_time)); + return; + } time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - elog(INFO, "delete: %s %s", + elog(INFO, "Delete: %s %s", base36enc(backup->start_time), timestamp); /* @@ -283,17 +286,17 @@ delete_backup_files(pgBackup *backup) pgFile *file = (pgFile *) parray_get(files, i); /* print progress */ - elog(VERBOSE, "delete file(%zd/%lu) \"%s\"", i + 1, + elog(VERBOSE, "Delete file(%zd/%lu) \"%s\"", i + 1, (unsigned long) parray_num(files), file->path); if (remove(file->path)) { - elog(WARNING, "can't remove \"%s\": %s", file->path, - strerror(errno)); - parray_walk(files, pgFileFree); - parray_free(files); - - return 1; + if (errno == ENOENT) + elog(VERBOSE, "File \"%s\" is absent", file->path); + else + elog(ERROR, "Cannot remove \"%s\": %s", file->path, + strerror(errno)); + return; } } @@ -301,7 +304,7 @@ delete_backup_files(pgBackup *backup) parray_free(files); backup->status = BACKUP_STATUS_DELETED; - return 0; + return; } /* diff --git a/src/merge.c b/src/merge.c index 137f1acd..3f32fff2 100644 --- a/src/merge.c +++ b/src/merge.c @@ -77,7 +77,8 @@ do_merge(time_t backup_id) { if (backup->status != BACKUP_STATUS_OK && /* It is possible that previous merging was interrupted */ - backup->status != BACKUP_STATUS_MERGING) + backup->status != BACKUP_STATUS_MERGING && + backup->status != BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", base36enc(backup->start_time), status2str(backup->status)); @@ -164,7 +165,14 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) int i; bool merge_isok = true; - elog(LOG, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + + /* + * Previous merging was interrupted during deleting source backup. It is + * safe just to delete it again. + */ + if (from_backup->status == BACKUP_STATUS_DELETING) + goto delete_source_backup; to_backup->status = BACKUP_STATUS_MERGING; write_backup_status(to_backup); @@ -243,68 +251,10 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) if (!merge_isok) elog(ERROR, "Data files merging failed"); - /* - * Files were copied into to_backup and deleted from from_backup. Remove - * remaining directories from from_backup. - */ - parray_qsort(files, pgFileComparePathDesc); - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - if (!S_ISDIR(file->mode)) - continue; - - if (rmdir(file->path)) - elog(ERROR, "Could not remove directory \"%s\": %s", - file->path, strerror(errno)); - } - if (rmdir(from_database_path)) - elog(ERROR, "Could not remove directory \"%s\": %s", - from_database_path, strerror(errno)); - if (unlink(control_file)) - elog(ERROR, "Could not remove file \"%s\": %s", - control_file, strerror(errno)); - - pgBackupGetPath(from_backup, control_file, lengthof(control_file), - BACKUP_CONTROL_FILE); - if (unlink(control_file)) - elog(ERROR, "Could not remove file \"%s\": %s", - control_file, strerror(errno)); - - if (rmdir(from_backup_path)) - elog(ERROR, "Could not remove directory \"%s\": %s", - from_backup_path, strerror(errno)); - - /* - * Delete files which are not in from_backup file list. - */ - for (i = 0; i < parray_num(to_files); i++) - { - pgFile *file = (pgFile *) parray_get(to_files, i); - - if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) - { - pgFileDelete(file); - elog(LOG, "Deleted \"%s\"", file->path); - } - } - - /* - * Rename FULL backup directory. - */ - if (rename(to_backup_path, from_backup_path) == -1) - elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", - to_backup_path, from_backup_path, strerror(errno)); - /* * Update to_backup metadata. */ - pgBackupCopy(to_backup, from_backup); - /* Correct metadata */ - to_backup->backup_mode = BACKUP_MODE_FULL; to_backup->status = BACKUP_STATUS_OK; - to_backup->parent_backup = INVALID_BACKUP_ID; /* Compute summary of size of regular files in the backup */ to_backup->data_bytes = 0; for (i = 0; i < parray_num(files); i++) @@ -325,7 +275,46 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) else to_backup->wal_bytes = BYTES_INVALID; - pgBackupWriteFileList(to_backup, files, from_database_path); + write_backup_filelist(to_backup, files, from_database_path); + write_backup(to_backup); + +delete_source_backup: + /* + * Files were copied into to_backup. It is time to remove source backup + * entirely. + */ + delete_backup_files(from_backup); + + /* + * Delete files which are not in from_backup file list. + */ + for (i = 0; i < parray_num(to_files); i++) + { + pgFile *file = (pgFile *) parray_get(to_files, i); + + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + elog(VERBOSE, "Deleted \"%s\"", file->path); + } + } + + /* + * Rename FULL backup directory. + */ + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Merging finished, now we can safely update ID of the destination backup. + */ + pgBackupCopy(to_backup, from_backup); + elog(INFO, "to_backup: %s", base36enc(to_backup->start_time)); + /* Correct metadata */ + to_backup->backup_mode = BACKUP_MODE_FULL; + to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; write_backup(to_backup); /* Cleanup */ @@ -508,10 +497,9 @@ merge_files(void *arg) file->write_size = pgFileSize(to_path_tmp); file->crc = pgFileGetCRC(to_path_tmp, false); } - pgFileDelete(file); } else - move_file(argument->from_root, argument->to_root, file); + copy_file(argument->from_root, argument->to_root, file); if (file->write_size != BYTES_INVALID) elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c6ff6343..cac1d747 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -448,6 +448,7 @@ extern int do_show(time_t requested_backup_id); /* in delete.c */ extern void do_delete(time_t backup_id); +extern void delete_backup_files(pgBackup *backup); extern int do_retention_purge(void); extern int do_delete_instance(void); @@ -478,10 +479,11 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli); extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); -extern void pgBackupWriteFileList(pgBackup *backup, parray *files, +extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root); -extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); +extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, + const char *subdir); extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); @@ -543,7 +545,6 @@ extern void restore_data_file(const char *to_path, bool write_header, uint32 backup_version); extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); -extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); extern void get_wal_file(const char *from_path, const char *to_path); diff --git a/tests/merge.py b/tests/merge.py index 0169b275..db0dc2c3 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -602,7 +602,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('move_file') + gdb.set_breakpoint('copy_file') gdb.run_until_break() if gdb.continue_execution_until_break(20) != 'breakpoint-hit': @@ -615,3 +615,10 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Try to continue failed MERGE self.merge_backup(backup_dir, "node", backup_id) + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 03c9fba7bd7157fad5f44d87a9ebe497ad9cb0f5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 24 Oct 2018 18:53:42 +0300 Subject: [PATCH 23/37] PGPRO-1892: Remove obsolete INFO message --- src/merge.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/merge.c b/src/merge.c index 3f32fff2..5d22002d 100644 --- a/src/merge.c +++ b/src/merge.c @@ -310,7 +310,6 @@ delete_source_backup: * Merging finished, now we can safely update ID of the destination backup. */ pgBackupCopy(to_backup, from_backup); - elog(INFO, "to_backup: %s", base36enc(to_backup->start_time)); /* Correct metadata */ to_backup->backup_mode = BACKUP_MODE_FULL; to_backup->status = BACKUP_STATUS_OK; From fcc4ed74392e727a8c4536a8b410e07a0e3202f8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Oct 2018 12:17:22 +0300 Subject: [PATCH 24/37] PGPRO-1892: added test_continue_failed_merge_with_corrupted_delta_backup --- tests/merge.py | 128 ++++++++++++++++++++++++++++++++++++++++- tests/validate_test.py | 11 ++++ 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index db0dc2c3..abd64565 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2,7 +2,7 @@ import unittest import os -from .helpers.ptrack_helpers import ProbackupTest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException module_name = "merge" @@ -622,3 +622,129 @@ class MergeTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_continue_failed_merge_with_corrupted_delta_backup(self): + """ + Fail merge via gdb, corrupt DELTA backup, try to continue merge + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i" + ) + + old_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500" + ) + + node.safe_psql( + "postgres", + "vacuum full t_heap" + ) + + new_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + # DELTA BACKUP + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + backup_id = self.show_pb(backup_dir, "node")[1]["id"] + + # Failed MERGE + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + if gdb.continue_execution_until_break(2) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + gdb._execute('signal SIGKILL') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + # CORRUPT incremental backup + # read block from future + # block_size + backup_header = 8200 + file = os.path.join( + backup_dir, 'backups/node', backup_id_2, 'database', new_path) + with open(file, 'rb') as f: + f.seek(8200) + block_1 = f.read(8200) + f.close + + # write block from future + file = os.path.join( + backup_dir, 'backups/node', backup_id, 'database', old_path) + with open(file, 'r+b') as f: + f.seek(8200) + f.write(block_1) + f.close + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of incremental backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'INSERT ERROR MESSAGE HERE\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# 1. always use parent link when merging (intermediates may be from different chain) +# 2. page backup we are merging with may disappear after failed merge, +# it should not be possible to continue merge after that +# PAGE_A MERGING (disappear) +# FULL MERGING + +# FULL MERGING + +# PAGE_B OK (new backup) +# FULL MERGING + +# 3. Need new test with corrupted FULL backup diff --git a/tests/validate_test.py b/tests/validate_test.py index b3590de3..ed3c7f10 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -3136,3 +3136,14 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.del_test_dir(module_name, fname) # validate empty backup list +# page from future during validate +# page from future during backup + +# corrupt block, so file become unaligned: +# 712 Assert(header.compressed_size <= BLCKSZ); +# 713 +# 714 read_len = fread(compressed_page.data, 1, +# 715 MAXALIGN(header.compressed_size), in); +# 716 if (read_len != MAXALIGN(header.compressed_size)) +# -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", +# 718 blknum, file->path, read_len, header.compressed_size); From ee6bab40a98cae698eb452f5ae050d6b91b2e9b0 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 9 Nov 2018 18:32:37 +0300 Subject: [PATCH 25/37] PGPRO-1892: Add validation of merged backups --- src/merge.c | 35 +++++++++++++++++++++++++++++++---- tests/merge.py | 14 +++++--------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/merge.c b/src/merge.c index 5d22002d..e3d6b9f8 100644 --- a/src/merge.c +++ b/src/merge.c @@ -59,7 +59,7 @@ do_merge(time_t backup_id) if (instance_name == NULL) elog(ERROR, "required parameter is not specified: --instance"); - elog(LOG, "Merge started"); + elog(INFO, "Merge started"); catalog_lock(); @@ -129,17 +129,21 @@ do_merge(time_t backup_id) */ for (i = full_backup_idx; i > dest_backup_idx; i--) { - pgBackup *to_backup = (pgBackup *) parray_get(backups, i); pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); - merge_backups(to_backup, from_backup); + full_backup = (pgBackup *) parray_get(backups, i); + merge_backups(full_backup, from_backup); } + pgBackupValidate(full_backup); + if (full_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Merging of backup %s failed", base36enc(backup_id)); + /* cleanup */ parray_walk(backups, pgBackupFree); parray_free(backups); - elog(LOG, "Merge completed"); + elog(INFO, "Merge of backup %s completed", base36enc(backup_id)); } /* @@ -167,6 +171,28 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + /* + * Validate to_backup only if it is BACKUP_STATUS_OK. If it has + * BACKUP_STATUS_MERGING status then it isn't valid backup until merging + * finished. + */ + if (to_backup->status == BACKUP_STATUS_OK) + { + pgBackupValidate(to_backup); + if (to_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Interrupt merging"); + } + + /* + * It is OK to validate from_backup if it has BACKUP_STATUS_OK or + * BACKUP_STATUS_MERGING status. + */ + Assert(from_backup->status == BACKUP_STATUS_OK || + from_backup->status == BACKUP_STATUS_MERGING); + pgBackupValidate(from_backup); + if (from_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Interrupt merging"); + /* * Previous merging was interrupted during deleting source backup. It is * safe just to delete it again. @@ -302,6 +328,7 @@ delete_source_backup: /* * Rename FULL backup directory. */ + elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id); if (rename(to_backup_path, from_backup_path) == -1) elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", to_backup_path, from_backup_path, strerror(errno)); diff --git a/tests/merge.py b/tests/merge.py index abd64565..826d19f1 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -694,8 +694,6 @@ class MergeTest(ProbackupTest, unittest.TestCase): gdb._execute('signal SIGKILL') - print(self.show_pb(backup_dir, as_text=True, as_json=False)) - # CORRUPT incremental backup # read block from future # block_size + backup_header = 8200 @@ -723,16 +721,14 @@ class MergeTest(ProbackupTest, unittest.TestCase): "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual( - e.message, - 'INSERT ERROR MESSAGE HERE\n', + self.assertTrue( + "WARNING: Backup {0} data files are corrupted".format( + backup_id) in e.message and + "ERROR: Merging of backup {0} failed".format( + backup_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - # Clean after yourself self.del_test_dir(module_name, fname) From 8e716791ba1f6942d93e7ddfd6e983663fe4bd3b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 11 Nov 2018 21:53:00 +0300 Subject: [PATCH 26/37] tests: minor fixes --- tests/compatibility.py | 159 ++++++++++++++++++++++++++ tests/compression.py | 67 +++++++++++ tests/replica.py | 248 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 464 insertions(+), 10 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 3d67bf3e..7c3e137f 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -311,3 +311,162 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_compression(self): + """Description in jira issue PGPRO-434""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # restore OLD FULL with new binary + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # PAGE backup with OLD binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='page', + old_binary=True, + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # PAGE backup with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='page', + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Delta backup with old binary + self.delete_pb(backup_dir, 'node', backup_id) + + self.backup_node( + backup_dir, 'node', node, + old_binary=True, + options=['--compress']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=['--compress'], + old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Delta backup with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/compression.py b/tests/compression.py index aa275382..54bc299c 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -494,3 +494,70 @@ class CompressionTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + def test_uncompressable_pages(self): + """ + make archive node, create table with uncompressable toast pages, + take backup with compression, make sure that page was not compressed, + restore backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + +# node.safe_psql( +# "postgres", +# "create table t_heap as select i, " +# "repeat('1234567890abcdefghiyklmn', 1)::bytea, " +# "point(0,0) from generate_series(0,1) i") + + node.safe_psql( + "postgres", + "create table t as select i, " + "repeat(md5(i::text),5006056) as fat_attr " + "from generate_series(0,10) i;") + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', + options=[ + '--compress', + '--log-level-file=verbose']) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', + options=[ + '--compress', + '--log-level-file=verbose']) + + # Clean after yourself + # self.del_test_dir(module_name, fname) + +# create table t as select i, repeat(md5('1234567890'), 1)::bytea, point(0,0) from generate_series(0,1) i; + + +# create table t_bytea_1(file oid); +# INSERT INTO t_bytea_1 (file) +# VALUES (lo_import('/home/gsmol/git/postgres/contrib/pg_probackup/tests/expected/sample.random', 24593)); +# insert into t_bytea select string_agg(data,'') from pg_largeobject where pageno > 0; +# \ No newline at end of file diff --git a/tests/replica.py b/tests/replica.py index d74c375c..f8c16607 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta import subprocess from sys import exit import time +from shutil import copyfile module_name = 'replica' @@ -64,6 +65,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): "from generate_series(256,512) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ @@ -80,9 +82,11 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): base_dir="{0}/{1}/node".format(module_name, fname)) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) node.slow_start() + # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -95,7 +99,9 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', options=[ @@ -111,9 +117,11 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): node.cleanup() self.restore_node( backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) node.slow_start() + # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -136,13 +144,12 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): pg_options={ 'wal_level': 'replica', 'max_wal_senders': '2', - 'checkpoint_timeout': '30s'} + 'checkpoint_timeout': '30s', + 'archive_timeout': '10s'} ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - # force more frequent wal switch - master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') master.slow_start() replica = self.make_simple_node( @@ -180,8 +187,14 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") self.add_instance(backup_dir, 'replica', replica) + + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000003'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000003')) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ @@ -201,9 +214,11 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) node.slow_start() + # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) + node.cleanup() # Change data on master, make PAGE backup from replica, # restore taken backup and check that restored data equal @@ -212,30 +227,41 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,768) i") + "from generate_series(512,22680) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( - backup_dir, 'replica', replica, backup_type='page', + backup_dir, 'replica', + replica, backup_type='page', options=[ '--archive-timeout=300', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) # RESTORE PAGE BACKUP TAKEN FROM replica - node.cleanup() self.restore_node( backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.append_conf( + 'postgresql.auto.conf', 'archive_mode = off') node.slow_start() + # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) + self.add_instance(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -279,15 +305,217 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node( backup_dir, 'master', master, backup_type='page') self.restore_node( - backup_dir, 'master', replica, - options=['-R', '--recovery-target-action=promote']) + backup_dir, 'master', replica, options=['-R']) # Settings for Replica - # self.set_replica(master, replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.start() + replica.append_conf( + 'postgresql.auto.conf', 'hot_standby = on') + + replica.slow_start(replica=True) + + self.add_instance(backup_dir, 'replica', replica) + + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000003'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000003')) + + self.backup_node(backup_dir, 'replica', replica) # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_take_backup_from_delayed_replica(self): + """ + make archive master, take full backups from master, + restore full backup as delayed replica, launch pgbench, + take FULL, PAGE and DELTA backups from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + #master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + self.restore_node( + backup_dir, 'master', replica, options=['-R']) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + # stupid hack + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000001'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000001')) + + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + + replica.append_conf( + 'postgresql.auto.conf', 'hot_standby = on') + + replica.append_conf( + 'recovery.conf', "recovery_min_apply_delay = '300s'") + + replica.slow_start(replica=True) + + master.pgbench_init(scale=10) + + pgbench = master.pgbench( + options=['-T', '30', '-c', '2', '--no-vacuum']) + + self.backup_node( + backup_dir, 'replica', replica) + + self.backup_node( + backup_dir, 'replica', replica, + data_dir=replica.data_dir, backup_type='page') + + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta') + + pgbench.wait() + + pgbench = master.pgbench( + options=['-T', '30', '-c', '2', '--no-vacuum']) + + self.backup_node( + backup_dir, 'replica', replica, + options=['--stream']) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='page', options=['--stream']) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + + pgbench.wait() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + def test_make_block_from_future(self): + """ + make archive master, take full backups from master, + restore full backup as replica, launch pgbench, + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + #master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + self.restore_node( + backup_dir, 'master', replica, options=['-R']) + + # Settings for Replica + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf( + 'postgresql.auto.conf', 'hot_standby = on') + + replica.slow_start(replica=True) + + self.add_instance(backup_dir, 'replica', replica) + + replica.safe_psql( + 'postgres', + 'checkpoint') + + master.pgbench_init(scale=10) + + self.wait_until_replica_catch_with_master(master, replica) + + +# print(replica.safe_psql( +# 'postgres', +# 'select * from pg_catalog.pg_last_xlog_receive_location()')) +# +# print(replica.safe_psql( +# 'postgres', +# 'select * from pg_catalog.pg_last_xlog_replay_location()')) +# +# print(replica.safe_psql( +# 'postgres', +# 'select * from pg_catalog.pg_control_checkpoint()')) +# +# replica.safe_psql( +# 'postgres', +# 'checkpoint') + + pgbench = master.pgbench(options=['-T', '30', '-c', '2', '--no-vacuum']) + + time.sleep(5) + + #self.backup_node(backup_dir, 'replica', replica, options=['--stream']) + exit(1) + self.backup_node(backup_dir, 'replica', replica, options=["--log-level-file=verbose"]) + pgbench.wait() + + # pgbench + master.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256000) i") + + + master.safe_psql( + 'postgres', + 'checkpoint') + + replica.safe_psql( + 'postgres', + 'checkpoint') + + replica.safe_psql( + 'postgres', + 'select * from pg_') + + self.backup_node(backup_dir, 'replica', replica) + exit(1) + + # Clean after yourself + self.del_test_dir(module_name, fname) \ No newline at end of file From d2271554a2173270857e57d715d1d64a07e6ebaf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 12 Nov 2018 11:51:58 +0300 Subject: [PATCH 27/37] tests: minor fixes --- tests/archive.py | 70 +++++++++++++++++++++++++++-------------- tests/backup_test.py | 2 +- tests/compatibility.py | 9 ++---- tests/compression.py | 10 ++---- tests/delta.py | 34 +++++++------------- tests/exclude.py | 2 +- tests/false_positive.py | 4 +-- tests/merge.py | 4 +-- tests/page.py | 11 +++---- tests/ptrack.py | 52 +++++++++++++----------------- tests/ptrack_clean.py | 2 +- tests/ptrack_empty.py | 2 +- tests/replica.py | 26 ++++++++++----- tests/validate_test.py | 24 ++++++-------- 14 files changed, 125 insertions(+), 127 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 8b8eb71a..4ed783d6 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta import subprocess from sys import exit from time import sleep +from shutil import copyfile module_name = 'archive' @@ -39,8 +40,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): result = node.safe_psql("postgres", "SELECT * FROM t_heap") self.backup_node( - backup_dir, 'node', node, - options=["--log-level-file=verbose"]) + backup_dir, 'node', node) node.cleanup() self.restore_node( @@ -53,8 +53,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Make backup self.backup_node( - backup_dir, 'node', node, - options=["--log-level-file=verbose"]) + backup_dir, 'node', node) node.cleanup() # Restore Database @@ -253,7 +252,6 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=[ "--archive-timeout=60", - "--log-level-file=verbose", "--stream"] ) # we should die here because exception is what we expect to happen @@ -402,7 +400,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.del_test_dir(module_name, fname) # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_replica_archive(self): """ make node without archiving, take stream backup and @@ -417,7 +415,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): initdb_params=['--data-checksums'], pg_options={ 'max_wal_senders': '2', - 'checkpoint_timeout': '30s', + 'archive_timeout': '10s', 'max_wal_size': '1GB'} ) self.init_pb(backup_dir) @@ -433,7 +431,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") + "from generate_series(0,2560) i") self.backup_node(backup_dir, 'master', master, options=['--stream']) before = master.safe_psql("postgres", "SELECT * FROM t_heap") @@ -459,9 +457,6 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") - # ADD INSTANCE 'REPLICA' - - sleep(1) backup_id = self.backup_node( backup_dir, 'replica', replica, @@ -469,7 +464,9 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): '--archive-timeout=30', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -493,16 +490,28 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,768) i") + "from generate_series(512,20680) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + master.safe_psql( + "postgres", + "CHECKPOINT") + +# copyfile( +# os.path.join(backup_dir, 'wal/master/000000010000000000000002'), +# os.path.join(backup_dir, 'wal/replica/000000010000000000000002')) + backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='page', options=[ - '--archive-timeout=30', '--log-level-file=verbose', - '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)] - ) + '--archive-timeout=30', + '--master-db=postgres', + '--master-host=localhost', + '--master-port={0}'.format(master.port), + '--stream']) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -511,8 +520,10 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node.cleanup() self.restore_node( backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") @@ -537,7 +548,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'} + 'archive_timeout': '10s'} ) replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -568,7 +579,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): pgdata_replica = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata_master, pgdata_replica) - self.set_replica(master, replica, synchronous=True) + self.set_replica(master, replica) # ADD INSTANCE REPLICA self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA @@ -579,16 +590,26 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0, 60000) i") + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000001'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000001')) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ - '--archive-timeout=20', - '--log-level-file=verbose', + '--archive-timeout=30', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)] - ) + '--master-port={0}'.format(master.port), + '--stream']) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -618,7 +639,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'} + 'checkpoint_timeout': '30s', + 'archive_timeout': '10s'} ) replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) diff --git a/tests/backup_test.py b/tests/backup_test.py index b21aab9b..9c8a0955 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -328,7 +328,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", '--log-level-file=verbose']) + options=["-j", "4", "--stream", "--log-level-file=verbose"]) # open log file and check with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: diff --git a/tests/compatibility.py b/tests/compatibility.py index 7c3e137f..39070b3f 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -94,8 +94,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -195,8 +194,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -296,8 +294,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/compression.py b/tests/compression.py index 54bc299c..2e712a15 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -55,9 +55,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=[ - '--stream', '--compress-algorithm=zlib', - '--log-level-console=verbose', - '--log-level-file=verbose']) + '--stream', '--compress-algorithm=zlib']) # PTRACK BACKUP node.safe_psql( @@ -535,8 +533,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type='full', options=[ - '--compress', - '--log-level-file=verbose']) + '--compress']) node.cleanup() @@ -547,8 +544,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type='full', options=[ - '--compress', - '--log-level-file=verbose']) + '--compress']) # Clean after yourself # self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py index bdbfac91..55cc03be 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -80,13 +80,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): pgdata = self.pgdata_content(node.data_dir) self.restore_node( - backup_dir, - 'node', - node_restored, - options=[ - "-j", "1", - "--log-level-file=verbose" - ] + backup_dir, 'node', node_restored ) # Physical comparison @@ -176,8 +170,6 @@ class DeltaTest(ProbackupTest, unittest.TestCase): 'node', node_restored, options=[ - "-j", "1", - "--log-level-file=verbose", "-T", "{0}={1}".format( old_tablespace, new_tablespace)] ) @@ -251,13 +243,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): pgdata = self.pgdata_content(node.data_dir) self.restore_node( - backup_dir, - 'node', - node_restored, - options=[ - "-j", "1", - "--log-level-file=verbose" - ] + backup_dir, 'node', node_restored ) # Physical comparison @@ -683,7 +669,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored, backup_id=backup_id, options=[ - "-j", "4", "--log-level-file=verbose", + "-j", "4", "--immediate", "--recovery-target-action=promote"]) @@ -717,7 +703,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored, backup_id=backup_id, options=[ - "-j", "4", "--log-level-file=verbose", + "-j", "4", "--immediate", "--recovery-target-action=promote"] ) @@ -815,7 +801,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', - options=["--stream", "--log-level-file=verbose"] + options=["--stream"] ) # if self.paranoia: # pgdata_delta = self.pgdata_content( @@ -844,7 +830,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): node_restored, backup_id=backup_id, options=[ - "-j", "4", "--log-level-file=verbose", + "-j", "4", "--immediate", "--recovery-target-action=promote"]) @@ -1135,7 +1121,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack_1(self): + def test_delta_corruption_heal_via_ptrack_1(self): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -1174,8 +1160,10 @@ class DeltaTest(ProbackupTest, unittest.TestCase): f.close self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=["-j", "4", "--stream", "--log-level-file=verbose"]) + backup_dir, 'node', node, + backup_type="delta", + options=["-j", "4", "--stream", '--log-level-file=verbose']) + # open log file and check with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: diff --git a/tests/exclude.py b/tests/exclude.py index 48b7889c..3fd3341f 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -143,7 +143,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/false_positive.py b/tests/false_positive.py index 04062b79..df7b1334 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -143,7 +143,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node, options=['--stream']) gdb = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'], + options=['--stream'], gdb=True ) @@ -227,7 +227,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node, options=['--stream']) gdb = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'], + options=['--stream'], gdb=True ) diff --git a/tests/merge.py b/tests/merge.py index 826d19f1..5f7ae7da 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -407,17 +407,17 @@ class MergeTest(ProbackupTest, unittest.TestCase): node.safe_psql( "postgres", "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( "postgres", "vacuum t_heap") - self.backup_node( + page_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - page_id = self.show_pb(backup_dir, "node")[1]["id"] self.merge_backup(backup_dir, "node", page_id) self.validate_pb(backup_dir) diff --git a/tests/page.py b/tests/page.py index 1d8238e1..d31b8f60 100644 --- a/tests/page.py +++ b/tests/page.py @@ -62,8 +62,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "vacuum t_heap") self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='page') self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -333,8 +332,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): result = node.safe_psql("postgres", "select * from pgbench_accounts") # PAGE BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=["--log-level-file=verbose"]) + backup_dir, 'node', node, backup_type='page') # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -727,7 +725,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type='page', - options=["-j", "4", '--log-level-file=verbose']) + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n " @@ -797,8 +795,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # Single-thread PAGE backup try: self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='page') self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n " diff --git a/tests/ptrack.py b/tests/ptrack.py index 72159318..5d01d882 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -157,13 +157,13 @@ class PtrackTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) pgdata = self.pgdata_content(node.data_dir) self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) self.restore_node( @@ -246,14 +246,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase): exit(1) self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--log-level-file=verbose'] - ) + backup_dir, 'node', node, backup_type='ptrack') self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--log-level-file=verbose'] - ) + backup_dir, 'node', node, backup_type='ptrack') + if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -336,14 +333,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--log-level-file=verbose'] - ) + backup_dir, 'node', node, backup_type='ptrack') self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--log-level-file=verbose'] - ) + backup_dir, 'node', node, backup_type='ptrack') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -409,7 +402,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) node.safe_psql( @@ -479,7 +472,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node, options=['--stream']) gdb = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'], + options=['--stream'], gdb=True ) @@ -566,7 +559,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ptrack_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) if self.paranoia: @@ -989,7 +982,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.safe_psql("postgres", "SELECT * FROM t_heap") self.backup_node( backup_dir, 'node', node, - options=["--stream", "--log-level-file=verbose"]) + options=["--stream"]) # CREATE DATABASE DB1 node.safe_psql("postgres", "create database db1") @@ -1002,7 +995,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "--log-level-file=verbose"] + options=["--stream"] ) if self.paranoia: @@ -1133,7 +1126,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): '-j10', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(node.port) + '--master-port={0}'.format(node.port), + '--stream' ] ) @@ -1229,7 +1223,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "--log-level-file=verbose"] + options=["--stream"] ) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1315,7 +1309,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", '--log-level-file=verbose']) + options=["--stream"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1476,7 +1470,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # FIRTS PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "--log-level-file=verbose"]) + options=["--stream"]) # GET PHYSICAL CONTENT FROM NODE if self.paranoia: @@ -1517,7 +1511,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # SECOND PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "--log-level-file=verbose"]) + options=["--stream"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1612,9 +1606,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): #result = node.safe_psql("postgres", "select * from pgbench_accounts") # FIRTS PTRACK BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=["--log-level-file=verbose"] - ) + backup_dir, 'node', node, backup_type='ptrack') + # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -1683,9 +1676,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=[ - "--stream", "-j 30", - "--log-level-file=verbose"] - ) + "--stream", "-j 30"]) + # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index ae16c662..076291a6 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -76,7 +76,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # Take PTRACK backup to clean every ptrack backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['-j10', '--log-level-file=verbose']) + options=['-j10']) node.safe_psql('postgres', 'checkpoint') for i in idx_ptrack: diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index 750a7336..8656f941 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -67,7 +67,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): # Take PTRACK backup backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['-j10', '--log-level-file=verbose']) + options=['-j10']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/replica.py b/tests/replica.py index f8c16607..1ab8515e 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -162,7 +162,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") + "from generate_series(0,2560) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") @@ -173,6 +173,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.set_replica(master, replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) # Check data correctness on replica @@ -186,7 +187,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,512) i") + "from generate_series(256,5120) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") self.add_instance(backup_dir, 'replica', replica) @@ -195,13 +196,23 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): os.path.join(backup_dir, 'wal/master/000000010000000000000003'), os.path.join(backup_dir, 'wal/replica/000000010000000000000003')) + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000004'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000004')) + + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000005'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000005')) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ - '--archive-timeout=300', + '--archive-timeout=30', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -235,10 +246,11 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', replica, backup_type='page', options=[ - '--archive-timeout=300', + '--archive-timeout=30', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) self.validate_pb(backup_dir, 'replica') self.assertEqual( @@ -491,7 +503,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): #self.backup_node(backup_dir, 'replica', replica, options=['--stream']) exit(1) - self.backup_node(backup_dir, 'replica', replica, options=["--log-level-file=verbose"]) + self.backup_node(backup_dir, 'replica', replica) pgbench.wait() # pgbench diff --git a/tests/validate_test.py b/tests/validate_test.py index ed3c7f10..c0fd4943 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -50,7 +50,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): f.close self.backup_node( - backup_dir, 'node', node, options=["--log-level-file=verbose"]) + backup_dir, 'node', node, options=['--log-level-file=verbose']) log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") @@ -259,8 +259,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Simple validate try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, - options=['--log-level-file=verbose']) + backup_dir, 'node', backup_id=backup_id_2) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -364,8 +363,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Validate PAGE1 try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, - options=['--log-level-file=verbose']) + backup_dir, 'node', backup_id=backup_id_2) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -520,8 +518,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): try: self.validate_pb( backup_dir, 'node', - backup_id=backup_id_4, - options=['--log-level-file=verbose']) + backup_id=backup_id_4) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n" @@ -721,7 +718,6 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.validate_pb( backup_dir, 'node', options=[ - '--log-level-file=verbose', '-i', backup_id_4, '--xid={0}'.format(target_xid)]) self.assertEqual( 1, 0, @@ -866,7 +862,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Validate Instance try: self.validate_pb( - backup_dir, 'node', options=['--log-level-file=verbose']) + backup_dir, 'node') self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -1006,7 +1002,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Validate Instance try: - self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: @@ -1092,7 +1088,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Validate Instance try: - self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: @@ -1219,7 +1215,6 @@ class ValidateTest(ProbackupTest, unittest.TestCase): 'node', backup_id, options=[ - "--log-level-console=verbose", "--xid={0}".format(target_xid)]) self.assertEqual( 1, 0, @@ -1388,7 +1383,6 @@ class ValidateTest(ProbackupTest, unittest.TestCase): 'node', backup_id, options=[ - "--log-level-console=verbose", "--xid={0}".format(target_xid)]) self.assertEqual( 1, 0, @@ -1671,7 +1665,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): os.rename(file_new, file) try: - self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + self.validate_pb(backup_dir) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid'.format( @@ -1776,7 +1770,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): os.rename(file, file_new) try: - self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + self.validate_pb(backup_dir) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid'.format( From 6eefeeba8dfe7280252d3abb1e6f424e0f2929ac Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 12 Nov 2018 12:15:54 +0300 Subject: [PATCH 28/37] tests: expected help fixed --- tests/expected/option_help.out | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 228598ed..ecc59a89 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -50,6 +50,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--master-db=db_name] [--master-host=host_name] [--master-port=port] [--master-user=user_name] [--replica-timeout=timeout] + [--skip-block-validation] pg_probackup restore -B backup-path --instance=instance_name [-D pgdata-path] [-i backup-id] [--progress] @@ -59,12 +60,14 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] [--no-validate] + [--skip-block-validation] pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] [--recovery-target-name=target-name] [--timeline=timeline] + [--skip-block-validation] pg_probackup show -B backup-path [--instance=instance_name [-i backup-id]] From de9c3b64d72693283b6514f0d0d5773613b3c971 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 12 Nov 2018 12:16:34 +0300 Subject: [PATCH 29/37] Version 2.0.24 - Major bugfix: incorrect handling of badly compressed blocks, previously there was a risk to restore block in uncompressed state, if compressed size was equal or larger than BLCKSZ - Impromevent: backup from replica >= 9.6 no longer need connection to master - Workaround: wrong minRecPoint in PostgreSQL thanks to commit 8d68ee6(block from future), overwrite minRecPoint with latest applied LSN - Impromevent: merge is now considered stable feature - Impromevent: validation now use more conservative and paranoid approach to file validation, during validation pg_probackup also check block checksumm, make sanity check based on block header information and try to detect blocks from future - New validate/restore options: '--skip-block-validation' - disable aforementioned approach to file validation - Multiple minor fixes --- src/pg_probackup.c | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ea99672d..00b0fc42 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -17,7 +17,7 @@ #include "utils/thread.h" -const char *PROGRAM_VERSION = "2.0.23"; +const char *PROGRAM_VERSION = "2.0.24"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 6a0391c2..5280a6f9 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.23 \ No newline at end of file +pg_probackup 2.0.24 \ No newline at end of file From 1e9615f56740d2e31ae693bd11234988b7db1627 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 12 Nov 2018 15:44:22 +0300 Subject: [PATCH 30/37] Use InvalidXLogRecPtr to mark infinite end, a couple code cleanup --- src/backup.c | 10 ++++++++-- src/pg_probackup.h | 1 - src/restore.c | 14 +++++++++----- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index 2317bee9..06796e15 100644 --- a/src/backup.c +++ b/src/backup.c @@ -474,6 +474,7 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; + parray *backup_list = NULL; pgFile *pg_control = NULL; @@ -515,7 +516,6 @@ do_backup_instance(void) current.backup_mode == BACKUP_MODE_DIFF_PTRACK || current.backup_mode == BACKUP_MODE_DIFF_DELTA) { - parray *backup_list; char prev_backup_filelist_path[MAXPGPATH]; /* get list of backups already taken */ @@ -525,7 +525,6 @@ do_backup_instance(void) if (prev_backup == NULL) elog(ERROR, "Valid backup on current timeline is not found. " "Create new FULL backup before an incremental one."); - parray_free(backup_list); pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); @@ -832,6 +831,13 @@ do_backup_instance(void) current.data_bytes += file->write_size; } + /* Cleanup */ + if (backup_list) + { + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + } + parray_walk(backup_files_list, pgFileFree); parray_free(backup_files_list); backup_files_list = NULL; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ca45d604..0fad6d70 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -418,7 +418,6 @@ extern int do_restore_or_validate(time_t target_backup_id, extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); -extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, diff --git a/src/restore.c b/src/restore.c index 439f3c4e..9cf33515 100644 --- a/src/restore.c +++ b/src/restore.c @@ -33,6 +33,7 @@ static void restore_backup(pgBackup *backup); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); +static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); @@ -138,7 +139,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); /* Read timeline history files from archives */ - timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); + timelines = read_timeline_history(rt->recovery_target_tli); if (!satisfy_timeline(timelines, current_backup)) { @@ -149,6 +150,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* Try to find another backup that satisfies target timeline */ continue; } + + parray_walk(timelines, pfree); + parray_free(timelines); } if (!satisfy_recovery_target(current_backup, rt)) @@ -731,7 +735,7 @@ create_recovery_conf(time_t backup_id, * based on readTimeLineHistory() in timeline.c */ parray * -readTimeLineHistory_probackup(TimeLineID targetTLI) +read_timeline_history(TimeLineID targetTLI) { parray *result; char path[MAXPGPATH]; @@ -820,8 +824,7 @@ readTimeLineHistory_probackup(TimeLineID targetTLI) entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; /* LSN in target timeline is valid */ - /* TODO ensure that -1UL --> -1L fix is correct */ - entry->end = (uint32) (-1L << 32) | -1L; + entry->end = InvalidXLogRecPtr; parray_insert(result, 0, entry); return result; @@ -853,7 +856,8 @@ satisfy_timeline(const parray *timelines, const pgBackup *backup) timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); if (backup->tli == timeline->tli && - backup->stop_lsn < timeline->end) + (XLogRecPtrIsInvalid(timeline->end) || + backup->stop_lsn < timeline->end)) return true; } return false; From 0a510f7211a45d6258039ca0a64d82615ba67f1c Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 13 Nov 2018 13:11:26 +0300 Subject: [PATCH 31/37] Some fixes for windows build --- src/backup.c | 1 + src/parsexlog.c | 1 + src/pg_probackup.c | 1 + src/utils/logger.c | 1 + src/utils/thread.c | 5 ++++- src/utils/thread.h | 6 ++++++ 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 06796e15..380dc1c0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -22,6 +22,7 @@ #include #include "utils/thread.h" +#include #define PG_STOP_BACKUP_TIMEOUT 300 diff --git a/src/parsexlog.c b/src/parsexlog.c index ee7b5076..ed73f70c 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -22,6 +22,7 @@ #endif #include "utils/thread.h" +#include /* * RmgrNames is an array of resource manager names, to make error messages diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 00b0fc42..e8240e19 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -16,6 +16,7 @@ #include #include "utils/thread.h" +#include const char *PROGRAM_VERSION = "2.0.24"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; diff --git a/src/utils/logger.c b/src/utils/logger.c index 4cdbf721..05f6fc5b 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -14,6 +14,7 @@ #include "logger.h" #include "pgut.h" #include "thread.h" +#include /* Logger parameters */ diff --git a/src/utils/thread.c b/src/utils/thread.c index 82c23764..0999a0d5 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -9,8 +9,11 @@ #include "thread.h" +#ifdef WIN32 +DWORD main_tid = 0; +#else pthread_t main_tid = 0; - +#endif #ifdef WIN32 #include diff --git a/src/utils/thread.h b/src/utils/thread.h index 06460533..6b8349bf 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -28,7 +28,13 @@ extern int pthread_join(pthread_t th, void **thread_return); #include #endif +#ifdef WIN32 +extern DWORD main_tid; +#else extern pthread_t main_tid; +#endif + + extern int pthread_lock(pthread_mutex_t *mp); From 5179f0219f9b79c835fd45f6548cd6224c8f3352 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 13 Nov 2018 13:46:29 +0300 Subject: [PATCH 32/37] Changed the script for creating the Windows project --- gen_probackup_project.pl | 75 ++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index e14f1d4b..d7f88c4d 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -1,10 +1,15 @@ # -*-perl-*- hey - emacs - this is a perl file -BEGIN{ +# my $currpath = cwd(); + +our $pgsrc; +our $currpath; + +BEGIN { +# path to the pg_pprobackup dir +$currpath = File::Basename::dirname(Cwd::abs_path($0)); use Cwd; use File::Basename; - -my $pgsrc=""; -if (@ARGV==1) +if (($#ARGV+1)==1) { $pgsrc = shift @ARGV; if($pgsrc eq "--help"){ @@ -23,14 +28,13 @@ else chdir($path); chdir("../.."); $pgsrc = cwd(); + $currpath = "contrib/pg_probackup"; } - chdir("$pgsrc/src/tools/msvc"); push(@INC, "$pgsrc/src/tools/msvc"); chdir("../../..") if (-d "../msvc" && -d "../../../src"); } - use Win32; use Carp; use strict; @@ -84,22 +88,27 @@ my $vcver = build_pgprobackup($config); my $bconf = $ENV{CONFIG} || "Release"; my $msbflags = $ENV{MSBFLAGS} || ""; my $buildwhat = $ARGV[1] || ""; -if (uc($ARGV[0]) eq 'DEBUG') -{ - $bconf = "Debug"; -} -elsif (uc($ARGV[0]) ne "RELEASE") -{ - $buildwhat = $ARGV[0] || ""; -} +# if (uc($ARGV[0]) eq 'DEBUG') +# { +# $bconf = "Debug"; +# } +# elsif (uc($ARGV[0]) ne "RELEASE") +# { +# $buildwhat = $ARGV[0] || ""; +# } + +# printf "currpath=$currpath"; + +# exit(0); # ... and do it system("msbuild pg_probackup.vcxproj /verbosity:normal $msbflags /p:Configuration=$bconf" ); - # report status my $status = $? >> 8; +printf("Status: $status\n"); +printf("Output file built in the folder $pgsrc/$bconf/pg_probackup\n"); exit $status; @@ -126,10 +135,10 @@ sub build_pgprobackup #vvs test my $probackup = - $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $solution->AddProject("pg_probackup", 'exe', "pg_probackup"); #, 'contrib/pg_probackup' $probackup->AddDefine('FRONTEND'); $probackup->AddFiles( - 'contrib/pg_probackup/src', + "$currpath/src", 'archive.c', 'backup.c', 'catalog.c', @@ -149,39 +158,39 @@ sub build_pgprobackup 'validate.c' ); $probackup->AddFiles( - 'contrib/pg_probackup/src/utils', + "$currpath/src/utils", 'json.c', 'logger.c', 'parray.c', 'pgut.c', 'thread.c' ); - $probackup->AddFile('src/backend/access/transam/xlogreader.c'); - $probackup->AddFile('src/backend/utils/hash/pg_crc.c'); + $probackup->AddFile("$pgsrc/src/backend/access/transam/xlogreader.c"); + $probackup->AddFile("$pgsrc/src/backend/utils/hash/pg_crc.c"); $probackup->AddFiles( - 'src/bin/pg_basebackup', + "$pgsrc/src/bin/pg_basebackup", 'receivelog.c', 'streamutil.c' ); - if (-e 'src/bin/pg_basebackup/walmethods.c') + if (-e "$pgsrc/src/bin/pg_basebackup/walmethods.c") { - $probackup->AddFile('src/bin/pg_basebackup/walmethods.c'); + $probackup->AddFile("$pgsrc/src/bin/pg_basebackup/walmethods.c"); } - $probackup->AddFile('src/bin/pg_rewind/datapagemap.c'); + $probackup->AddFile("$pgsrc/src/bin/pg_rewind/datapagemap.c"); - $probackup->AddFile('src/interfaces/libpq/pthread-win32.c'); + $probackup->AddFile("$pgsrc/src/interfaces/libpq/pthread-win32.c"); - $probackup->AddIncludeDir('src/bin/pg_basebackup'); - $probackup->AddIncludeDir('src/bin/pg_rewind'); - $probackup->AddIncludeDir('src/interfaces/libpq'); - $probackup->AddIncludeDir('src'); - $probackup->AddIncludeDir('src/port'); + $probackup->AddIncludeDir("$pgsrc/src/bin/pg_basebackup"); + $probackup->AddIncludeDir("$pgsrc/src/bin/pg_rewind"); + $probackup->AddIncludeDir("$pgsrc/src/interfaces/libpq"); + $probackup->AddIncludeDir("$pgsrc/src"); + $probackup->AddIncludeDir("$pgsrc/src/port"); - $probackup->AddIncludeDir('contrib/pg_probackup'); - $probackup->AddIncludeDir('contrib/pg_probackup/src'); - $probackup->AddIncludeDir('contrib/pg_probackup/src/utils'); + $probackup->AddIncludeDir("$currpath"); + $probackup->AddIncludeDir("$currpath/src"); + $probackup->AddIncludeDir("$currpath/src/utils"); $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); $probackup->AddLibrary('ws2_32.lib'); From a769f47217f742bf0356d5602a25bd18c021a3ed Mon Sep 17 00:00:00 2001 From: Victor Wagner Date: Tue, 13 Nov 2018 15:03:26 +0300 Subject: [PATCH 33/37] Fix compilation under FreeBSD and Solaris --- src/data.c | 2 ++ src/parsexlog.c | 1 + src/util.c | 2 ++ 3 files changed, 5 insertions(+) diff --git a/src/data.c b/src/data.c index 9b6cbd23..b1acfecc 100644 --- a/src/data.c +++ b/src/data.c @@ -14,6 +14,8 @@ #include "storage/checksum_impl.h" #include +#include + #include #ifdef HAVE_LIBZ diff --git a/src/parsexlog.c b/src/parsexlog.c index ed73f70c..86827a85 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -22,6 +22,7 @@ #endif #include "utils/thread.h" +#include #include /* diff --git a/src/util.c b/src/util.c index 94a18a9a..f2820650 100644 --- a/src/util.c +++ b/src/util.c @@ -16,6 +16,8 @@ #include +#include + const char * base36enc(long unsigned int value) { From 4a1ca601afec4939f2e7e5395ff5137c4d1bcda5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 13 Nov 2018 15:49:09 +0300 Subject: [PATCH 34/37] PGPRO-2160: to_files may be uninitialized if from_backup has BACKUP_STATUS_DELETING --- src/merge.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/merge.c b/src/merge.c index e3d6b9f8..13263c64 100644 --- a/src/merge.c +++ b/src/merge.c @@ -164,7 +164,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) control_file[MAXPGPATH]; parray *files, *to_files; - pthread_t *threads; + pthread_t *threads = NULL; merge_files_arg *threads_args; int i; bool merge_isok = true; @@ -193,19 +193,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) if (from_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Interrupt merging"); - /* - * Previous merging was interrupted during deleting source backup. It is - * safe just to delete it again. - */ - if (from_backup->status == BACKUP_STATUS_DELETING) - goto delete_source_backup; - - to_backup->status = BACKUP_STATUS_MERGING; - write_backup_status(to_backup); - - from_backup->status = BACKUP_STATUS_MERGING; - write_backup_status(from_backup); - /* * Make backup paths. */ @@ -216,8 +203,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), DATABASE_DIR); - create_data_directories(to_database_path, from_backup_path, false); - /* * Get list of files which will be modified or removed. */ @@ -238,6 +223,21 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* sort by size for load balancing */ parray_qsort(files, pgFileCompareSize); + /* + * Previous merging was interrupted during deleting source backup. It is + * safe just to delete it again. + */ + if (from_backup->status == BACKUP_STATUS_DELETING) + goto delete_source_backup; + + to_backup->status = BACKUP_STATUS_MERGING; + write_backup_status(to_backup); + + from_backup->status = BACKUP_STATUS_MERGING; + write_backup_status(from_backup); + + create_data_directories(to_database_path, from_backup_path, false); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); @@ -344,8 +344,11 @@ delete_source_backup: write_backup(to_backup); /* Cleanup */ - pfree(threads_args); - pfree(threads); + if (threads) + { + pfree(threads_args); + pfree(threads); + } parray_walk(to_files, pgFileFree); parray_free(to_files); From 8505e78c9139fbe6fa82e8bdaa1d1043436247c3 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 13 Nov 2018 18:15:19 +0300 Subject: [PATCH 35/37] Bug fix: do not add root slash for pg_wal path --- src/parsexlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 86827a85..5b2e32af 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -536,8 +536,8 @@ validate_wal(pgBackup *backup, const char *archivedir, */ if (backup->stream) { - snprintf(backup_xlog_path, sizeof(backup_xlog_path), "/%s/%s/%s/%s", - backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); + pgBackupGetPath2(backup, backup_xlog_path, lengthof(backup_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, seg_size); From 0e445a99e8749719f3c68fad074ea650c7e5a8e0 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Wed, 14 Nov 2018 18:10:57 +0300 Subject: [PATCH 36/37] Removed obsolete scripts and project templates for Windows --- doit.cmd | 4 +- msvs/pg_probackup.sln | 28 ---- msvs/template.pg_probackup.vcxproj | 213 ------------------------ msvs/template.pg_probackup96.vcxproj | 211 ----------------------- msvs/template.pg_probackup_2.vcxproj | 204 ----------------------- win32build.pl | 240 --------------------------- win32build96.pl | 240 --------------------------- win32build_2.pl | 219 ------------------------ 8 files changed, 3 insertions(+), 1356 deletions(-) delete mode 100644 msvs/pg_probackup.sln delete mode 100644 msvs/template.pg_probackup.vcxproj delete mode 100644 msvs/template.pg_probackup96.vcxproj delete mode 100644 msvs/template.pg_probackup_2.vcxproj delete mode 100644 win32build.pl delete mode 100644 win32build96.pl delete mode 100644 win32build_2.pl diff --git a/doit.cmd b/doit.cmd index b46e3b36..7830a7ea 100644 --- a/doit.cmd +++ b/doit.cmd @@ -1 +1,3 @@ -perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file +CALL "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64 +SET PERL5LIB=. +perl gen_probackup_project.pl C:\Shared\Postgresql\myPostgres\11\postgrespro \ No newline at end of file diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln deleted file mode 100644 index 2df4b404..00000000 --- a/msvs/pg_probackup.sln +++ /dev/null @@ -1,28 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Express 2013 for Windows Desktop -VisualStudioVersion = 12.0.31101.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj deleted file mode 100644 index a0a3c7a8..00000000 --- a/msvs/template.pg_probackup.vcxproj +++ /dev/null @@ -1,213 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {4886B21A-D8CA-4A03-BADF-743B24C88327} - Win32Proj - pg_probackup - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - %(AdditionalLibraryDirectories) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - %(AdditionalLibraryDirectories) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj deleted file mode 100644 index 3c62734e..00000000 --- a/msvs/template.pg_probackup96.vcxproj +++ /dev/null @@ -1,211 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {4886B21A-D8CA-4A03-BADF-743B24C88327} - Win32Proj - pg_probackup - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - %(AdditionalLibraryDirectories) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - %(AdditionalLibraryDirectories) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj deleted file mode 100644 index 1f103ac8..00000000 --- a/msvs/template.pg_probackup_2.vcxproj +++ /dev/null @@ -1,204 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {4886B21A-D8CA-4A03-BADF-743B24C88327} - Win32Proj - pg_probackup - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/win32build.pl b/win32build.pl deleted file mode 100644 index 14864181..00000000 --- a/win32build.pl +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/perl -use JSON; -our $repack_version; -our $pgdir; -our $pgsrc; -if (@ARGV!=2) { - print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; - exit 1; -} - - -our $liblist=""; - - -$pgdir = shift @ARGV; -$pgsrc = shift @ARGV if @ARGV; - - -our $arch = $ENV{'ARCH'} || "x64"; -$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); -$arch='x64' if $arch eq 'X64'; - -$conffile = $pgsrc."/tools/msvc/config.pl"; - - -die 'Could not find config.pl' - unless (-f $conffile); - -our $config; -do $conffile; - - -if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { - print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; - exit 1; -} -our $includepath=""; -our $libpath=""; -our $libpath32=""; -AddProject(); - -print "\n\n"; -print $libpath."\n"; -print $includepath."\n"; - -# open F,"<","META.json" or die "Cannot open META.json: $!\n"; -# { -# local $/ = undef; -# $decoded = decode_json(); -# $repack_version= $decoded->{'version'}; -# } - -# substitute new path in the project files - - - -preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); - -exit 0; - - -sub preprocess_project { - my $in = shift; - my $out = shift; - our $pgdir; - our $adddir; - my $libs; - if (defined $adddir) { - $libs ="$adddir;"; - } else{ - $libs =""; - } - open IN,"<",$in or die "Cannot open $in: $!\n"; - open OUT,">",$out or die "Cannot open $out: $!\n"; - -# $includepath .= ";"; -# $libpath .= ";"; - - while () { - s/\@PGROOT\@/$pgdir/g; - s/\@ADDLIBS\@/$libpath/g; - s/\@ADDLIBS32\@/$libpath32/g; - s/\@PGSRC\@/$pgsrc/g; - s/\@ADDINCLUDE\@/$includepath/g; - - - print OUT $_; - } - close IN; - close OUT; - -} - - - -# my sub -sub AddLibrary -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - if ($libpath32 ne '') - { - $libpath32 .= ';'; - } - $libpath32 .= $inc; - -} -sub AddLibrary32 -{ - $inc = shift; - if ($libpath32 ne '') - { - $libpath32 .= ';'; - } - $libpath32 .= $inc; - -} -sub AddLibrary64 -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - -} - -sub AddIncludeDir -{ - # my ($self, $inc) = @_; - $inc = shift; - if ($includepath ne '') - { - $includepath .= ';'; - } - $includepath .= $inc; - -} - -sub AddProject -{ - # my ($self, $name, $type, $folder, $initialdir) = @_; - - if ($config->{zlib}) - { - AddIncludeDir($config->{zlib} . '\include'); - AddLibrary($config->{zlib} . '\lib\zdll.lib'); - } - if ($config->{openssl}) - { - AddIncludeDir($config->{openssl} . '\include'); - if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") - { - AddLibrary( - $config->{openssl} . '\lib\VC\ssleay32.lib', 1); - AddLibrary( - $config->{openssl} . '\lib\VC\libeay32.lib', 1); - } - else - { - # We don't expect the config-specific library to be here, - # so don't ask for it in last parameter - AddLibrary( - $config->{openssl} . '\lib\ssleay32.lib', 0); - AddLibrary( - $config->{openssl} . '\lib\libeay32.lib', 0); - } - } - if ($config->{nls}) - { - AddIncludeDir($config->{nls} . '\include'); - AddLibrary($config->{nls} . '\lib\libintl.lib'); - } - if ($config->{gss}) - { - AddIncludeDir($config->{gss} . '\inc\krb5'); - AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); - AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); - AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); - } - if ($config->{iconv}) - { - AddIncludeDir($config->{iconv} . '\include'); - AddLibrary($config->{iconv} . '\lib\iconv.lib'); - } - if ($config->{icu}) - { - AddIncludeDir($config->{icu} . '\include'); - AddLibrary32($config->{icu} . '\lib\icuin.lib'); - AddLibrary32($config->{icu} . '\lib\icuuc.lib'); - AddLibrary32($config->{icu} . '\lib\icudt.lib'); - AddLibrary64($config->{icu} . '\lib64\icuin.lib'); - AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); - AddLibrary64($config->{icu} . '\lib64\icudt.lib'); - } - if ($config->{xml}) - { - AddIncludeDir($config->{xml} . '\include'); - AddIncludeDir($config->{xml} . '\include\libxml2'); - AddLibrary($config->{xml} . '\lib\libxml2.lib'); - } - if ($config->{xslt}) - { - AddIncludeDir($config->{xslt} . '\include'); - AddLibrary($config->{xslt} . '\lib\libxslt.lib'); - } - if ($config->{libedit}) - { - AddIncludeDir($config->{libedit} . '\include'); - # AddLibrary($config->{libedit} . "\\" . - # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); - AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); - AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); - - - } - if ($config->{uuid}) - { - AddIncludeDir($config->{uuid} . '\include'); - AddLibrary($config->{uuid} . '\lib\uuid.lib'); - } - - if ($config->{zstd}) - { - AddIncludeDir($config->{zstd}); - # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); - AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); - AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; - } - # return $proj; -} - - - - diff --git a/win32build96.pl b/win32build96.pl deleted file mode 100644 index c869e485..00000000 --- a/win32build96.pl +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/perl -use JSON; -our $repack_version; -our $pgdir; -our $pgsrc; -if (@ARGV!=2) { - print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; - exit 1; -} - - -our $liblist=""; - - -$pgdir = shift @ARGV; -$pgsrc = shift @ARGV if @ARGV; - - -our $arch = $ENV{'ARCH'} || "x64"; -$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); -$arch='x64' if $arch eq 'X64'; - -$conffile = $pgsrc."/tools/msvc/config.pl"; - - -die 'Could not find config.pl' - unless (-f $conffile); - -our $config; -do $conffile; - - -if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { - print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; - exit 1; -} -our $includepath=""; -our $libpath=""; -our $libpath32=""; -AddProject(); - -print "\n\n"; -print $libpath."\n"; -print $includepath."\n"; - -# open F,"<","META.json" or die "Cannot open META.json: $!\n"; -# { -# local $/ = undef; -# $decoded = decode_json(); -# $repack_version= $decoded->{'version'}; -# } - -# substitute new path in the project files - - - -preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); - -exit 0; - - -sub preprocess_project { - my $in = shift; - my $out = shift; - our $pgdir; - our $adddir; - my $libs; - if (defined $adddir) { - $libs ="$adddir;"; - } else{ - $libs =""; - } - open IN,"<",$in or die "Cannot open $in: $!\n"; - open OUT,">",$out or die "Cannot open $out: $!\n"; - -# $includepath .= ";"; -# $libpath .= ";"; - - while () { - s/\@PGROOT\@/$pgdir/g; - s/\@ADDLIBS\@/$libpath/g; - s/\@ADDLIBS32\@/$libpath32/g; - s/\@PGSRC\@/$pgsrc/g; - s/\@ADDINCLUDE\@/$includepath/g; - - - print OUT $_; - } - close IN; - close OUT; - -} - - - -# my sub -sub AddLibrary -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - if ($libpath32 ne '') - { - $libpath32 .= ';'; - } - $libpath32 .= $inc; - -} -sub AddLibrary32 -{ - $inc = shift; - if ($libpath32 ne '') - { - $libpath32 .= ';'; - } - $libpath32 .= $inc; - -} -sub AddLibrary64 -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - -} - -sub AddIncludeDir -{ - # my ($self, $inc) = @_; - $inc = shift; - if ($includepath ne '') - { - $includepath .= ';'; - } - $includepath .= $inc; - -} - -sub AddProject -{ - # my ($self, $name, $type, $folder, $initialdir) = @_; - - if ($config->{zlib}) - { - AddIncludeDir($config->{zlib} . '\include'); - AddLibrary($config->{zlib} . '\lib\zdll.lib'); - } - if ($config->{openssl}) - { - AddIncludeDir($config->{openssl} . '\include'); - if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") - { - AddLibrary( - $config->{openssl} . '\lib\VC\ssleay32.lib', 1); - AddLibrary( - $config->{openssl} . '\lib\VC\libeay32.lib', 1); - } - else - { - # We don't expect the config-specific library to be here, - # so don't ask for it in last parameter - AddLibrary( - $config->{openssl} . '\lib\ssleay32.lib', 0); - AddLibrary( - $config->{openssl} . '\lib\libeay32.lib', 0); - } - } - if ($config->{nls}) - { - AddIncludeDir($config->{nls} . '\include'); - AddLibrary($config->{nls} . '\lib\libintl.lib'); - } - if ($config->{gss}) - { - AddIncludeDir($config->{gss} . '\inc\krb5'); - AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); - AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); - AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); - } - if ($config->{iconv}) - { - AddIncludeDir($config->{iconv} . '\include'); - AddLibrary($config->{iconv} . '\lib\iconv.lib'); - } - if ($config->{icu}) - { - AddIncludeDir($config->{icu} . '\include'); - AddLibrary32($config->{icu} . '\lib\icuin.lib'); - AddLibrary32($config->{icu} . '\lib\icuuc.lib'); - AddLibrary32($config->{icu} . '\lib\icudt.lib'); - AddLibrary64($config->{icu} . '\lib64\icuin.lib'); - AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); - AddLibrary64($config->{icu} . '\lib64\icudt.lib'); - } - if ($config->{xml}) - { - AddIncludeDir($config->{xml} . '\include'); - AddIncludeDir($config->{xml} . '\include\libxml2'); - AddLibrary($config->{xml} . '\lib\libxml2.lib'); - } - if ($config->{xslt}) - { - AddIncludeDir($config->{xslt} . '\include'); - AddLibrary($config->{xslt} . '\lib\libxslt.lib'); - } - if ($config->{libedit}) - { - AddIncludeDir($config->{libedit} . '\include'); - # AddLibrary($config->{libedit} . "\\" . - # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); - AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); - AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); - - - } - if ($config->{uuid}) - { - AddIncludeDir($config->{uuid} . '\include'); - AddLibrary($config->{uuid} . '\lib\uuid.lib'); - } - - if ($config->{zstd}) - { - AddIncludeDir($config->{zstd}); - # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); - AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); - AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; - } - # return $proj; -} - - - - diff --git a/win32build_2.pl b/win32build_2.pl deleted file mode 100644 index a4f75553..00000000 --- a/win32build_2.pl +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/perl -use JSON; -our $repack_version; -our $pgdir; -our $pgsrc; -if (@ARGV!=2) { - print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; - exit 1; -} - - -our $liblist=""; - - -$pgdir = shift @ARGV; -$pgsrc = shift @ARGV if @ARGV; - - -our $arch = $ENV{'ARCH'} || "x64"; -$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); -$arch='x64' if $arch eq 'X64'; - -$conffile = $pgsrc."/tools/msvc/config.pl"; - - -die 'Could not find config.pl' - unless (-f $conffile); - -our $config; -do $conffile; - - -if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { - print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; - exit 1; -} -our $includepath=""; -our $libpath=""; -AddProject(); - -print "\n\n"; -print $libpath."\n"; -print $includepath."\n"; - -# open F,"<","META.json" or die "Cannot open META.json: $!\n"; -# { -# local $/ = undef; -# $decoded = decode_json(); -# $repack_version= $decoded->{'version'}; -# } - -# substitute new path in the project files - - - -preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); - -exit 0; - - -sub preprocess_project { - my $in = shift; - my $out = shift; - our $pgdir; - our $adddir; - my $libs; - if (defined $adddir) { - $libs ="$adddir;"; - } else{ - $libs =""; - } - open IN,"<",$in or die "Cannot open $in: $!\n"; - open OUT,">",$out or die "Cannot open $out: $!\n"; - -# $includepath .= ";"; -# $libpath .= ";"; - - while () { - s/\@PGROOT\@/$pgdir/g; - s/\@ADDLIBS\@/$libpath/g; - s/\@PGSRC\@/$pgsrc/g; - s/\@ADDINCLUDE\@/$includepath/g; - - - print OUT $_; - } - close IN; - close OUT; - -} - - - -# my sub -sub AddLibrary -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - -} -sub AddIncludeDir -{ - # my ($self, $inc) = @_; - $inc = shift; - if ($includepath ne '') - { - $includepath .= ';'; - } - $includepath .= $inc; - -} - -sub AddProject -{ - # my ($self, $name, $type, $folder, $initialdir) = @_; - - if ($config->{zlib}) - { - AddIncludeDir($config->{zlib} . '\include'); - AddLibrary($config->{zlib} . '\lib\zdll.lib'); - } - if ($config->{openssl}) - { - AddIncludeDir($config->{openssl} . '\include'); - if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") - { - AddLibrary( - $config->{openssl} . '\lib\VC\ssleay32.lib', 1); - AddLibrary( - $config->{openssl} . '\lib\VC\libeay32.lib', 1); - } - else - { - # We don't expect the config-specific library to be here, - # so don't ask for it in last parameter - AddLibrary( - $config->{openssl} . '\lib\ssleay32.lib', 0); - AddLibrary( - $config->{openssl} . '\lib\libeay32.lib', 0); - } - } - if ($config->{nls}) - { - AddIncludeDir($config->{nls} . '\include'); - AddLibrary($config->{nls} . '\lib\libintl.lib'); - } - if ($config->{gss}) - { - AddIncludeDir($config->{gss} . '\inc\krb5'); - AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); - AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); - AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); - } - if ($config->{iconv}) - { - AddIncludeDir($config->{iconv} . '\include'); - AddLibrary($config->{iconv} . '\lib\iconv.lib'); - } - if ($config->{icu}) - { - AddIncludeDir($config->{icu} . '\include'); - if ($arch eq 'Win32') - { - AddLibrary($config->{icu} . '\lib\icuin.lib'); - AddLibrary($config->{icu} . '\lib\icuuc.lib'); - AddLibrary($config->{icu} . '\lib\icudt.lib'); - } - else - { - AddLibrary($config->{icu} . '\lib64\icuin.lib'); - AddLibrary($config->{icu} . '\lib64\icuuc.lib'); - AddLibrary($config->{icu} . '\lib64\icudt.lib'); - } - } - if ($config->{xml}) - { - AddIncludeDir($config->{xml} . '\include'); - AddIncludeDir($config->{xml} . '\include\libxml2'); - AddLibrary($config->{xml} . '\lib\libxml2.lib'); - } - if ($config->{xslt}) - { - AddIncludeDir($config->{xslt} . '\include'); - AddLibrary($config->{xslt} . '\lib\libxslt.lib'); - } - if ($config->{libedit}) - { - AddIncludeDir($config->{libedit} . '\include'); - AddLibrary($config->{libedit} . "\\" . - ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); - } - if ($config->{uuid}) - { - AddIncludeDir($config->{uuid} . '\include'); - AddLibrary($config->{uuid} . '\lib\uuid.lib'); - } - if ($config->{libedit}) - { - AddIncludeDir($config->{libedit} . '\include'); - AddLibrary($config->{libedit} . "\\" . - ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); - } - if ($config->{zstd}) - { - AddIncludeDir($config->{zstd}); - AddLibrary($config->{zstd}. "\\". - ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") - ); - } - # return $proj; -} - - - - From d04d314ebab7ddafaed9ca8bc1c73d3dadba5c6c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 15 Nov 2018 11:46:42 +0300 Subject: [PATCH 37/37] Keep compiler quite --- src/backup.c | 4 ++-- src/merge.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 380dc1c0..dbafd7e1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2173,7 +2173,7 @@ backup_files(void *arg) if (S_ISREG(buf.st_mode)) { - pgFile **prev_file; + pgFile **prev_file = NULL; /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) @@ -2214,7 +2214,7 @@ backup_files(void *arg) bool skip = false; /* If non-data file has not changed since last backup... */ - if (file->exists_in_prev && + if (prev_file && file->exists_in_prev && buf.st_mtime < current.parent_backup) { calc_file_checksum(file); diff --git a/src/merge.c b/src/merge.c index 13263c64..c4d3a22f 100644 --- a/src/merge.c +++ b/src/merge.c @@ -165,7 +165,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) parray *files, *to_files; pthread_t *threads = NULL; - merge_files_arg *threads_args; + merge_files_arg *threads_args = NULL; int i; bool merge_isok = true;