db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-30 09:47:53 +02:00
Code Issues Releases Activity

PBCKP-604: Allow partial incremental restore only with a flag --destroy-all-other-dbs

Browse Source
This commit is contained in:
Yuriy Sokolov 2023-05-25 18:12:11 +03:00 committed by Yura Sokolov
parent 0690f8d10e
commit c9fc20b898
7 changed files with 170 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/help.c
View File

@ -175,6 +175,7 @@ help_pg_probackup(void)
printf(_(" [-X WALDIR | --waldir=WALDIR]\n"));
printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n"));
printf(_(" [--db-include | --db-exclude]\n"));
printf(_(" [--destroy-all-other-dbs]\n"));
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
printf(_(" [--ssh-options]\n"));
@ -450,6 +451,7 @@ help_restore(void)
printf(_(" [-X WALDIR | --waldir=WALDIR]\n"));
printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n"));
printf(_(" [--db-include dbname | --db-exclude dbname]\n"));
printf(_(" [--destroy-all-other-dbs]\n"));
printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n"));
printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n"));
printf(_(" [--recovery-target-timeline=timeline]\n"));
@ -497,6 +499,9 @@ help_restore(void)
printf(_("\n Partial restore options:\n"));
printf(_(" --db-include dbname restore only specified databases\n"));
printf(_(" --db-exclude dbname do not restore specified databases\n"));
printf(_(" --destroy-all-other-dbs\n"));
printf(_(" allows to do partial restore that is prohibited by default,\n"));
printf(_(" because it might remove all other databases.\n"));
printf(_("\n Recovery options:\n"));
printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n"));

3
src/pg_probackup.c
View File

@ -124,6 +124,7 @@ static parray *datname_include_list = NULL;
static parray *exclude_absolute_paths_list = NULL;
static parray *exclude_relative_paths_list = NULL;
static char* gl_waldir_path = NULL;
static bool allow_partial_incremental = false;
/* checkdb options */
bool need_amcheck = false;
@ -242,6 +243,7 @@ static ConfigOption cmd_options[] =
{ 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT },
{ 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT },
{ 's', 'X', "waldir", &gl_waldir_path, SOURCE_CMD_STRICT },
{ 'b', 242, "destroy-all-other-dbs", &allow_partial_incremental, SOURCE_CMD_STRICT },
/* checkdb options */
{ 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT },
{ 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT },
@ -764,6 +766,7 @@ main(int argc, char *argv[])
restore_params->partial_restore_type = NONE;
restore_params->primary_conninfo = primary_conninfo;
restore_params->incremental_mode = incremental_mode;
restore_params->allow_partial_incremental = allow_partial_incremental;
/* handle partial restore parameters */
if (datname_exclude_list && datname_include_list)

8
src/pg_probackup.h
View File

@ -179,6 +179,7 @@ typedef enum DestDirIncrCompatibility
POSTMASTER_IS_RUNNING,
SYSTEM_ID_MISMATCH,
BACKUP_LABEL_EXISTS,
PARTIAL_INCREMENTAL_FORBIDDEN,
DEST_IS_NOT_OK,
DEST_OK
} DestDirIncrCompatibility;
@ -585,7 +586,8 @@ typedef struct pgRestoreParams
/* options for partial restore */
PartialRestoreType partial_restore_type;
parray *partial_db_list;
bool allow_partial_incremental;
char* waldir;
} pgRestoreParams;
@ -903,7 +905,9 @@ extern parray *get_backup_filelist(pgBackup *backup, bool strict);
extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict);
extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli);
extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
IncrRestoreMode incremental_mode);
IncrRestoreMode incremental_mode,
parray *partial_db_list,
bool allow_partial_incremental);
/* in remote.c */
extern void check_remote_agent_compatibility(int agent_version,

25
src/restore.c
View File

@ -150,6 +150,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
if (params->incremental_mode != INCR_NONE)
{
DestDirIncrCompatibility rc;
const char *message = NULL;
bool ok_to_go = true;
elog(INFO, "Running incremental restore into nonempty directory: \"%s\"",
@ -157,12 +158,15 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
rc = check_incremental_compatibility(instance_config.pgdata,
instance_config.system_identifier,
params->incremental_mode);
params->incremental_mode,
params->partial_db_list,
params->allow_partial_incremental);
if (rc == POSTMASTER_IS_RUNNING)
{
/* Even with force flag it is unwise to run
* incremental restore over running instance
*/
message = "Postmaster is running.";
ok_to_go = false;
}
else if (rc == SYSTEM_ID_MISMATCH)
@ -174,7 +178,10 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
if (params->incremental_mode != INCR_NONE && params->force)
cleanup_pgdata = true;
else
{
message = "System ID mismatch.";
ok_to_go = false;
}
}
else if (rc == BACKUP_LABEL_EXISTS)
{
@ -187,7 +194,10 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
* to calculate switchpoint.
*/
if (params->incremental_mode == INCR_LSN)
{
message = "Backup label exists. Cannot use incremental restore in LSN mode.";
ok_to_go = false;
}
}
else if (rc == DEST_IS_NOT_OK)
{
@ -196,11 +206,16 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg
* so we cannot be sure that postmaster is running or not.
* It is better to just error out.
*/
message = "We cannot be sure about the database state.";
ok_to_go = false;
} else if (rc == PARTIAL_INCREMENTAL_FORBIDDEN)
{
message = "Partial incremental restore into non-empty PGDATA is forbidden.";
ok_to_go = false;
}
if (!ok_to_go)
elog(ERROR, "Incremental restore is not allowed");
elog(ERROR, "Incremental restore is not allowed: %s", message);
}
else
elog(ERROR, "Restore destination is not empty: \"%s\"",
@ -2142,7 +2157,9 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
*/
DestDirIncrCompatibility
check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
IncrRestoreMode incremental_mode)
IncrRestoreMode incremental_mode,
parray *partial_db_list,
bool allow_partial_incremental)
{
uint64 system_id_pgdata;
bool system_id_match = false;
@ -2226,6 +2243,8 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
if (backup_label_exists)
return BACKUP_LABEL_EXISTS;
if (partial_db_list && !allow_partial_incremental)
return PARTIAL_INCREMENTAL_FORBIDDEN;
/* some other error condition */
if (!success)
return DEST_IS_NOT_OK;

1
tests/expected/option_help.out
View File

@ -92,6 +92,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[-X WALDIR | --waldir=WALDIR]
[-I | --incremental-mode=none|checksum|lsn]
[--db-include | --db-exclude]
[--destroy-all-other-dbs]
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]

1
tests/expected/option_help_ru.out
View File

@ -92,6 +92,7 @@ pg_probackup - утилита для управления резервным к
[-X WALDIR | --waldir=WALDIR]
[-I | --incremental-mode=none|checksum|lsn]
[--db-include | --db-exclude]
[--destroy-all-other-dbs]
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]

136
tests/incr_restore_test.py
View File

@ -1962,7 +1962,9 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
node2, options=[
"--db-exclude=db1",
"--db-exclude=db5",
"-I", "checksum"])
"-I", "checksum",
"--destroy-all-other-dbs",
])
pgdata2 = self.pgdata_content(node2.data_dir)
@ -2068,7 +2070,9 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
node2, options=[
"--db-exclude=db1",
"--db-exclude=db5",
"-I", "lsn"])
"-I", "lsn",
"--destroy-all-other-dbs",
])
pgdata2 = self.pgdata_content(node2.data_dir)
@ -2188,7 +2192,8 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
"--db-exclude=db1",
"--db-exclude=db5",
"-T", "{0}={1}".format(
node_tablespace, node2_tablespace)])
node_tablespace, node2_tablespace),
"--destroy-all-other-dbs"])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
@ -2209,7 +2214,9 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
"--db-exclude=db1",
"--db-exclude=db5",
"-T", "{0}={1}".format(
node_tablespace, node2_tablespace)])
node_tablespace, node2_tablespace),
"--destroy-all-other-dbs",
])
pgdata2 = self.pgdata_content(node2.data_dir)
@ -2241,6 +2248,127 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
self.assertNotIn('PANIC', output)
def test_incremental_partial_restore_deny(self):
"""
Do now allow partial incremental restore into non-empty PGDATA
becase we can't limit WAL replay to a single database.
"""
backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(self.module_name, self.fname, 'node'),
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
for i in range(1, 3):
node.safe_psql('postgres', f'CREATE database db{i}')
# FULL backup
backup_id = self.backup_node(backup_dir, 'node', node)
pgdata = self.pgdata_content(node.data_dir)
try:
self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN'])
self.fail("incremental partial restore is not allowed")
except ProbackupException as e:
self.assertIn("Incremental restore is not allowed: Postmaster is running.", e.message)
node.safe_psql('db2', 'create table x (id int)')
node.safe_psql('db2', 'insert into x values (42)')
node.stop()
try:
self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN'])
self.fail("because incremental partial restore is not allowed")
except ProbackupException as e:
self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message)
node.slow_start()
value = node.execute('db2', 'select * from x')[0][0]
self.assertEqual(42, value)
def test_deny_incremental_partial_restore_exclude_tablespace_checksum(self):
"""
Do now allow partial incremental restore into non-empty PGDATA
becase we can't limit WAL replay to a single database.
(case of tablespaces)
"""
backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(self.module_name, self.fname, 'node'),
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
self.create_tblspace_in_node(node, 'somedata')
node_tablespace = self.get_tblspace_path(node, 'somedata')
tbl_oid = node.safe_psql(
'postgres',
"SELECT oid "
"FROM pg_tablespace "
"WHERE spcname = 'somedata'").rstrip()
for i in range(1, 10, 1):
node.safe_psql(
'postgres',
'CREATE database db{0} tablespace somedata'.format(i))
db_list_raw = node.safe_psql(
'postgres',
'SELECT to_json(a) '
'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
db_list_splitted = db_list_raw.splitlines()
db_list = {}
for line in db_list_splitted:
line = json.loads(line)
db_list[line['datname']] = line['oid']
# FULL backup
backup_id = self.backup_node(backup_dir, 'node', node)
# node2
node2 = self.make_simple_node('node2')
node2.cleanup()
node2_tablespace = self.get_tblspace_path(node2, 'somedata')
# in node2 restore full backup
self.restore_node(
backup_dir, 'node',
node2, options=[
"-T", f"{node_tablespace}={node2_tablespace}"])
# partial incremental restore into node2
try:
self.restore_node(backup_dir, 'node', node2,
options=["-I", "checksum",
"--db-exclude=db1",
"--db-exclude=db5",
"-T", f"{node_tablespace}={node2_tablespace}"])
self.fail("remapped tablespace contain old data")
except ProbackupException as e:
pass
try:
self.restore_node(backup_dir, 'node', node2,
options=[
"-I", "checksum", "--force",
"--db-exclude=db1", "--db-exclude=db5",
"-T", f"{node_tablespace}={node2_tablespace}"])
self.fail("incremental partial restore is not allowed")
except ProbackupException as e:
self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message)
def test_incremental_pg_filenode_map(self):
"""
https://github.com/postgrespro/pg_probackup/issues/320