tests: more tests for partial restore

tests/restore.py

@@ -2429,8 +2429,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
             self.restore_node(
                 backup_dir, 'node',
                 node_restored_1, options=[
-                    "--db-include='db1'",
+                    "--db-include=db1",
-                    "--db-exclude='db2'"])
+                    "--db-exclude=db2"])
             self.assertEqual(
                 1, 0,
                 "Expecting Error because of 'db-exclude' and 'db-include'.\n "
@@ -2670,8 +2670,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
             self.restore_node(
                 backup_dir, 'node',
                 node_restored_1, options=[
-                    "--db-include='db1'",
+                    "--db-include=db1",
-                    "--db-exclude='db2'"])
+                    "--db-exclude=db2"])
             self.assertEqual(
                 1, 0,
                 "Expecting Error because of 'db-exclude' and 'db-include'.\n "
@@ -2753,10 +2753,436 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-# partial restore
+    def test_partial_restore_backward_compatibility_1(self):
-# 0. basic test for db-exclude and db-include +
+        """
-# 1. old backup without support of partial restore
+        """
-# 2. FULL backup do not support partial restore, but incremental do
+        fname = self.id().split('.')[3]
-# 3. database_map is missing for legal reasons, e.g. no permissions for pg_database
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
-# 4. database_map is empty for illegal reason
+        node = self.make_simple_node(
-# 5. database_map contain garbage
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'])
+
+        self.init_pb(backup_dir, old_binary=True)
+        self.add_instance(backup_dir, 'node', node, old_binary=True)
+
+        node.slow_start()
+
+        # create databases
+        for i in range(1, 10, 1):
+            node.safe_psql(
+                'postgres',
+                'CREATE database db{0}'.format(i))
+
+        # FULL backup with old binary, without partial restore support
+        backup_id = self.backup_node(
+            backup_dir, 'node', node,
+            old_binary=True, options=['--stream'])
+
+        pgdata = self.pgdata_content(node.data_dir)
+
+        node_restored = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node_restored'))
+        node_restored.cleanup()
+
+        try:
+            self.restore_node(
+                backup_dir, 'node',
+                node_restored, options=[
+                    "--db-exclude=db5"])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because backup do not support partial restore.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Backup {0} has missing database_map".format(backup_id),
+                e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        self.restore_node(backup_dir, 'node', node_restored)
+
+        pgdata_restored = self.pgdata_content(node_restored.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
+
+        # incremental backup with partial restore support
+        for i in range(11, 15, 1):
+            node.safe_psql(
+                'postgres',
+                'CREATE database db{0}'.format(i))
+
+        # get db list
+        db_list_raw = node.safe_psql(
+            'postgres',
+            'SELECT to_json(a) '
+            'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
+        db_list_splitted = db_list_raw.splitlines()
+        db_list = {}
+        for line in db_list_splitted:
+            line = json.loads(line)
+            db_list[line['datname']] = line['oid']
+
+        backup_id = self.backup_node(
+            backup_dir, 'node', node,
+            backup_type='delta', options=['--stream'])
+
+        # get etalon
+        node_restored.cleanup()
+        self.restore_node(backup_dir, 'node', node_restored)
+        self.truncate_every_file_in_dir(
+                os.path.join(
+                    node_restored.data_dir, 'base', db_list['db5']))
+        self.truncate_every_file_in_dir(
+                os.path.join(
+                    node_restored.data_dir, 'base', db_list['db14']))
+        pgdata_restored = self.pgdata_content(node_restored.data_dir)
+
+        # get new node
+        node_restored_1 = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node_restored_1'))
+        node_restored_1.cleanup()
+
+        self.restore_node(
+                backup_dir, 'node',
+                node_restored_1, options=[
+                    "--db-exclude=db5",
+                    "--db-exclude=db14"])
+        pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir)
+
+        self.compare_pgdata(pgdata_restored, pgdata_restored_1)
+
+    def test_partial_restore_backward_compatibility_merge(self):
+        """
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'])
+
+        self.init_pb(backup_dir, old_binary=True)
+        self.add_instance(backup_dir, 'node', node, old_binary=True)
+
+        node.slow_start()
+
+        # create databases
+        for i in range(1, 10, 1):
+            node.safe_psql(
+                'postgres',
+                'CREATE database db{0}'.format(i))
+
+        # FULL backup with old binary, without partial restore support
+        backup_id = self.backup_node(
+            backup_dir, 'node', node,
+            old_binary=True, options=['--stream'])
+
+        pgdata = self.pgdata_content(node.data_dir)
+
+        node_restored = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node_restored'))
+        node_restored.cleanup()
+
+        try:
+            self.restore_node(
+                backup_dir, 'node',
+                node_restored, options=[
+                    "--db-exclude=db5"])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because backup do not support partial restore.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Backup {0} has missing database_map".format(backup_id),
+                e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        self.restore_node(backup_dir, 'node', node_restored)
+
+        pgdata_restored = self.pgdata_content(node_restored.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
+
+        # incremental backup with partial restore support
+        for i in range(11, 15, 1):
+            node.safe_psql(
+                'postgres',
+                'CREATE database db{0}'.format(i))
+
+        # get db list
+        db_list_raw = node.safe_psql(
+            'postgres',
+            'SELECT to_json(a) '
+            'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
+        db_list_splitted = db_list_raw.splitlines()
+        db_list = {}
+        for line in db_list_splitted:
+            line = json.loads(line)
+            db_list[line['datname']] = line['oid']
+
+        backup_id = self.backup_node(
+            backup_dir, 'node', node,
+            backup_type='delta', options=['--stream'])
+
+        # get etalon
+        node_restored.cleanup()
+        self.restore_node(backup_dir, 'node', node_restored)
+        self.truncate_every_file_in_dir(
+                os.path.join(
+                    node_restored.data_dir, 'base', db_list['db5']))
+        self.truncate_every_file_in_dir(
+                os.path.join(
+                    node_restored.data_dir, 'base', db_list['db14']))
+        pgdata_restored = self.pgdata_content(node_restored.data_dir)
+
+        # get new node
+        node_restored_1 = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node_restored_1'))
+        node_restored_1.cleanup()
+
+        # merge
+        self.merge_backup(backup_dir, 'node', backup_id=backup_id)
+
+        self.restore_node(
+                backup_dir, 'node',
+                node_restored_1, options=[
+                    "--db-exclude=db5",
+                    "--db-exclude=db14"])
+        pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir)
+
+        self.compare_pgdata(pgdata_restored, pgdata_restored_1)
+
+    def test_missing_database_map(self):
+        """
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'])
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+
+        node.slow_start()
+
+        # create databases
+        for i in range(1, 10, 1):
+            node.safe_psql(
+                'postgres',
+                'CREATE database db{0}'.format(i))
+
+        node.safe_psql(
+            "postgres",
+            "CREATE DATABASE backupdb")
+
+        if self.get_version(node) > self.version_to_num('10.0'):
+            # bootstrap for 10/11
+            node.safe_psql(
+                "backupdb",
+                "REVOKE ALL on SCHEMA public from public; "
+                "REVOKE ALL on SCHEMA pg_catalog from public; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM public; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public; "
+                "CREATE ROLE backup WITH LOGIN REPLICATION; "
+                "GRANT CONNECT ON DATABASE postgres to backup; "
+                "GRANT USAGE ON SCHEMA pg_catalog TO backup; "
+                "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
+                # we use it for partial restore and checkdb
+                # "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; "
+                # "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, oid) TO backup;"
+            )
+        else:
+            # bootstrap for 9.5/9.6
+            node.safe_psql(
+                "backupdb",
+                "REVOKE ALL on SCHEMA public from public; "
+                "REVOKE ALL on SCHEMA pg_catalog from public; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM public; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public; "
+                "CREATE ROLE backup WITH LOGIN REPLICATION; "
+                "GRANT CONNECT ON DATABASE postgres to backup; "
+                "GRANT USAGE ON SCHEMA pg_catalog TO backup; "
+                # we use it for ptrack
+                "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
+                # we use it for partial restore and checkdb
+                # "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; "
+                # "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, oid) TO backup;"
+            )
+
+        # FULL backup without database_map
+        backup_id = self.backup_node(
+            backup_dir, 'node', node, datname='backupdb',
+            options=['--stream', "-U", "backup"])
+
+        pgdata = self.pgdata_content(node.data_dir)
+
+        node_restored = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node_restored'))
+        node_restored.cleanup()
+
+        # backup has missing database_map and that is legal
+        try:
+            self.restore_node(
+                backup_dir, 'node', node_restored,
+                options=["--db-exclude=db5", "--db-exclude=db9"])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because user do not have pg_database access.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Backup {0} has missing database_map, "
+                "partial restore is impossible.".format(
+                    backup_id), e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        try:
+            self.restore_node(
+                backup_dir, 'node', node_restored,
+                options=["--db-include=db1"])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because user do not have pg_database access.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Backup {0} has missing database_map, "
+                "partial restore is impossible.".format(
+                    backup_id), e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        # check that simple restore is still possible
+        self.restore_node(backup_dir, 'node', node_restored)
+
+        pgdata_restored = self.pgdata_content(node_restored.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
+
+    def test_empty_and_mangled_database_map(self):
+        """
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'])
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+
+        node.slow_start()
+
+        # create databases
+        for i in range(1, 10, 1):
+            node.safe_psql(
+                'postgres',
+                'CREATE database db{0}'.format(i))
+
+        # FULL backup with database_map
+        backup_id = self.backup_node(
+            backup_dir, 'node', node, options=['--stream'])
+        pgdata = self.pgdata_content(node.data_dir)
+
+        # truncate database_map
+        path = os.path.join(
+            backup_dir, 'backups', 'node',
+            backup_id, 'database', 'database_map')
+        with open(path, "w") as f:
+            f.close()
+
+        node_restored = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node_restored'))
+        node_restored.cleanup()
+
+        try:
+            self.restore_node(
+                backup_dir, 'node', node_restored,
+                options=["--db-include=db1", '--no-validate'])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because database_map is empty.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Backup {0} has empty or mangled database_map, "
+                "partial restore is impossible".format(backup_id), e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        try:
+            self.restore_node(
+                backup_dir, 'node', node_restored,
+                options=["--db-exclude=db1", '--no-validate'])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because database_map is empty.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Backup {0} has empty or mangled database_map, "
+                "partial restore is impossible".format(backup_id), e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        # mangle database_map
+        with open(path, "w") as f:
+            f.write("42")
+            f.close()
+
+        try:
+            self.restore_node(
+                backup_dir, 'node', node_restored,
+                options=["--db-include=db1", '--no-validate'])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because database_map is empty.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                'ERROR: field "dbOid" is not found in the line 42 of '
+                'the file backup_content.control', e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        # check that simple restore is still possible
+        self.restore_node(
+            backup_dir, 'node', node_restored, options=['--no-validate'])
+
+        pgdata_restored = self.pgdata_content(node_restored.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)