db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-04-13 12:00:41 +02:00
Code Issues Releases Activity

Merge pull request #515 from postgrespro/PBCKP-170

Browse Source 
PBCKP-170
This commit is contained in:
dlepikhova 2022-08-04 20:56:34 +05:00 committed by GitHub
commit 2bc096bdf0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 415 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/backup.c
View File

@ -695,7 +695,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
*/
int
do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params,
bool no_validate, bool no_sync, bool backup_logs)
bool no_validate, bool no_sync, bool backup_logs, time_t start_time)
{
PGconn *backup_conn = NULL;
PGNodeInfo nodeInfo;
@ -710,7 +710,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params,
current.external_dir_str = instance_config.external_dir_str;
/* Create backup directory and BACKUP_CONTROL_FILE */
pgBackupCreateDir(&current, instanceState->instance_backup_subdir_path);
pgBackupCreateDir(&current, instanceState, start_time);
if (!instance_config.pgdata)
elog(ERROR, "required parameter not specified: PGDATA "

35
src/catalog.c
View File

@ -23,7 +23,7 @@ static pgBackup* get_closest_backup(timelineInfo *tlinfo);
static pgBackup* get_oldest_backup(timelineInfo *tlinfo);
static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"};
static pgBackup *readBackupControlFile(const char *path);
static time_t create_backup_dir(pgBackup *backup, const char *backup_instance_path);
static void create_backup_dir(pgBackup *backup, const char *backup_instance_path);
static bool backup_lock_exit_hook_registered = false;
static parray *locks = NULL;
@ -1420,10 +1420,12 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list,
*/
void
pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path)
pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time)
{
int i;
parray *subdirs = parray_new();
parray * backups;
pgBackup *target_backup;
parray_append(subdirs, pg_strdup(DATABASE_DIR));
@ -1444,7 +1446,26 @@ pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path)
free_dir_list(external_list);
}
backup->backup_id = create_backup_dir(backup, backup_instance_path);
/* Get list of all backups*/
backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID);
if (parray_num(backups) > 0)
{
target_backup = (pgBackup *) parray_get(backups, 0);
if (start_time > target_backup->backup_id)
{
backup->backup_id = start_time;
create_backup_dir(backup, instanceState->instance_backup_subdir_path);
}
else
{
elog(ERROR, "Cannot create directory for older backup");
}
}
else
{
backup->backup_id = start_time;
create_backup_dir(backup, instanceState->instance_backup_subdir_path);
}
if (backup->backup_id == 0)
elog(ERROR, "Cannot create backup directory: %s", strerror(errno));
@ -1471,7 +1492,7 @@ pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path)
* Create root directory for backup,
* update pgBackup.root_dir if directory creation was a success
*/
time_t
void
create_backup_dir(pgBackup *backup, const char *backup_instance_path)
{
int attempts = 10;
@ -1480,9 +1501,8 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path)
{
int rc;
char path[MAXPGPATH];
time_t backup_id = time(NULL);
join_path_components(path, backup_instance_path, base36enc(backup_id));
join_path_components(path, backup_instance_path, base36enc(backup->backup_id));
/* TODO: add wrapper for remote mode */
rc = dir_create_dir(path, DIR_PERMISSION, true);
@ -1490,7 +1510,7 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path)
if (rc == 0)
{
backup->root_dir = pgut_strdup(path);
return backup_id;
return;
}
else
{
@ -1499,7 +1519,6 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path)
}
}
return 0;
}
/*

11
src/pg_probackup.c
View File

@ -78,6 +78,7 @@ pid_t my_pid = 0;
__thread int my_thread_num = 1;
bool progress = false;
bool no_sync = false;
time_t start_time = 0;
#if PG_VERSION_NUM >= 100000
char *replication_slot = NULL;
bool temp_slot = false;
@ -201,6 +202,7 @@ static ConfigOption cmd_options[] =
{ 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT },
{ 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT },
{ 'b', 134, "no-color", &no_color, SOURCE_CMD_STRICT },
{ 'U', 241, "start-time", &start_time, SOURCE_CMD_STRICT },
/* backup options */
{ 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT },
{ 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT },
@ -957,6 +959,13 @@ main(int argc, char *argv[])
case BACKUP_CMD:
{
current.stream = stream_wal;
if (start_time == 0)
start_time = current_time;
else
elog(WARNING, "Please do not use the --start-time option to start backup. "
"This is a service option required to work with other extensions. "
"We do not guarantee future support for this flag.");
/* sanity */
if (current.backup_mode == BACKUP_MODE_INVALID)
@ -964,7 +973,7 @@ main(int argc, char *argv[])
"(-b, --backup-mode)");
return do_backup(instanceState, set_backup_params,
no_validate, no_sync, backup_logs);
no_validate, no_sync, backup_logs, start_time);
}
case CATCHUP_CMD:
return do_catchup(catchup_source_pgdata, catchup_destination_pgdata, num_threads, !no_sync,

4
src/pg_probackup.h
View File

@ -842,7 +842,7 @@ extern char** commands_args;
/* in backup.c */
extern int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params,
bool no_validate, bool no_sync, bool backup_logs);
bool no_validate, bool no_sync, bool backup_logs, time_t start_time);
extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt,
char *pgdata);
extern BackupMode parse_backup_mode(const char *value);
@ -983,7 +983,7 @@ extern void write_backup_filelist(pgBackup *backup, parray *files,
const char *root, parray *external_list, bool sync);
extern void pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path);
extern void pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time);
extern void pgNodeInit(PGNodeInfo *node);
extern void pgBackupInit(pgBackup *backup);
extern void pgBackupFree(void *backup);

371
tests/backup.py
View File

@ -1,6 +1,6 @@
import unittest
import os
from time import sleep
from time import sleep, time
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
import shutil
from distutils.dir_util import copy_tree
@ -3421,3 +3421,372 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_start_time(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# FULL backup
startTime = int(time())
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=['--stream', '--start-time', str(startTime)])
# DELTA backup
startTime = int(time())
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=['--stream', '--start-time', str(startTime)])
# PAGE backup
startTime = int(time())
self.backup_node(
backup_dir, 'node', node, backup_type="page",
options=['--stream', '--start-time', str(startTime)])
if self.ptrack and node.major_version > 11:
node.safe_psql(
"postgres",
"create extension ptrack")
# PTRACK backup
startTime = int(time())
self.backup_node(
backup_dir, 'node', node, backup_type="ptrack",
options=['--stream', '--start-time', str(startTime)])
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_start_time_incorrect_time(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
startTime = int(time())
#backup with correct start time
self.backup_node(
backup_dir, 'node', node,
options=['--stream', '--start-time', str(startTime)])
#backups with incorrect start time
try:
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=['--stream', '--start-time', str(startTime-10000)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because start time for new backup must be newer "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertRegex(
e.message,
"ERROR: Cannot create directory for older backup",
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
try:
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=['--stream', '--start-time', str(startTime-10000)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because start time for new backup must be newer "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertRegex(
e.message,
"ERROR: Cannot create directory for older backup",
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
try:
self.backup_node(
backup_dir, 'node', node, backup_type="page",
options=['--stream', '--start-time', str(startTime-10000)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because start time for new backup must be newer "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertRegex(
e.message,
"ERROR: Cannot create directory for older backup",
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
if self.ptrack and node.major_version > 11:
node.safe_psql(
"postgres",
"create extension ptrack")
try:
self.backup_node(
backup_dir, 'node', node, backup_type="page",
options=['--stream', '--start-time', str(startTime-10000)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because start time for new backup must be newer "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertRegex(
e.message,
"ERROR: Cannot create directory for older backup",
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_start_time_few_nodes(self):
fname = self.id().split('.')[3]
node1 = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node1'),
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'])
backup_dir1 = os.path.join(self.tmp_path, module_name, fname, 'backup1')
self.init_pb(backup_dir1)
self.add_instance(backup_dir1, 'node1', node1)
self.set_archiving(backup_dir1, 'node1', node1)
node1.slow_start()
node2 = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node2'),
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'])
backup_dir2 = os.path.join(self.tmp_path, module_name, fname, 'backup2')
self.init_pb(backup_dir2)
self.add_instance(backup_dir2, 'node2', node2)
self.set_archiving(backup_dir2, 'node2', node2)
node2.slow_start()
# FULL backup
startTime = int(time())
self.backup_node(
backup_dir1, 'node1', node1, backup_type="full",
options=['--stream', '--start-time', str(startTime)])
self.backup_node(
backup_dir2, 'node2', node2, backup_type="full",
options=['--stream', '--start-time', str(startTime)])
show_backup1 = self.show_pb(backup_dir1, 'node1')[0]
show_backup2 = self.show_pb(backup_dir2, 'node2')[0]
self.assertEqual(show_backup1['id'], show_backup2['id'])
# DELTA backup
startTime = int(time())
self.backup_node(
backup_dir1, 'node1', node1, backup_type="delta",
options=['--stream', '--start-time', str(startTime)])
self.backup_node(
backup_dir2, 'node2', node2, backup_type="delta",
options=['--stream', '--start-time', str(startTime)])
show_backup1 = self.show_pb(backup_dir1, 'node1')[1]
show_backup2 = self.show_pb(backup_dir2, 'node2')[1]
self.assertEqual(show_backup1['id'], show_backup2['id'])
# PAGE backup
startTime = int(time())
self.backup_node(
backup_dir1, 'node1', node1, backup_type="page",
options=['--stream', '--start-time', str(startTime)])
self.backup_node(
backup_dir2, 'node2', node2, backup_type="page",
options=['--stream', '--start-time', str(startTime)])
show_backup1 = self.show_pb(backup_dir1, 'node1')[2]
show_backup2 = self.show_pb(backup_dir2, 'node2')[2]
self.assertEqual(show_backup1['id'], show_backup2['id'])
# PTRACK backup
startTime = int(time())
if self.ptrack and node1.major_version > 11:
node1.safe_psql(
"postgres",
"create extension ptrack")
self.backup_node(
backup_dir1, 'node1', node1, backup_type="ptrack",
options=['--stream', '--start-time', str(startTime)])
if self.ptrack and node2.major_version > 11:
node2.safe_psql(
"postgres",
"create extension ptrack")
self.backup_node(
backup_dir2, 'node2', node2, backup_type="ptrack",
options=['--stream', '--start-time', str(startTime)])
show_backup1 = self.show_pb(backup_dir1, 'node1')[3]
show_backup2 = self.show_pb(backup_dir2, 'node2')[3]
self.assertEqual(show_backup1['id'], show_backup2['id'])
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_start_time_few_nodes_incorrect_time(self):
fname = self.id().split('.')[3]
node1 = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node1'),
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'])
backup_dir1 = os.path.join(self.tmp_path, module_name, fname, 'backup1')
self.init_pb(backup_dir1)
self.add_instance(backup_dir1, 'node1', node1)
self.set_archiving(backup_dir1, 'node1', node1)
node1.slow_start()
node2 = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node2'),
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'])
backup_dir2 = os.path.join(self.tmp_path, module_name, fname, 'backup2')
self.init_pb(backup_dir2)
self.add_instance(backup_dir2, 'node2', node2)
self.set_archiving(backup_dir2, 'node2', node2)
node2.slow_start()
# FULL backup
startTime = int(time())
self.backup_node(
backup_dir1, 'node1', node1, backup_type="full",
options=['--stream', '--start-time', str(startTime)])
self.backup_node(
backup_dir2, 'node2', node2, backup_type="full",
options=['--stream', '--start-time', str(startTime-10000)])
show_backup1 = self.show_pb(backup_dir1, 'node1')[0]
show_backup2 = self.show_pb(backup_dir2, 'node2')[0]
self.assertGreater(show_backup1['id'], show_backup2['id'])
# DELTA backup
startTime = int(time())
self.backup_node(
backup_dir1, 'node1', node1, backup_type="delta",
options=['--stream', '--start-time', str(startTime)])
# make backup with start time definitelly earlier, than existing
try:
self.backup_node(
backup_dir2, 'node2', node2, backup_type="delta",
options=['--stream', '--start-time', str(10000)])
self.assertEqual(
1, 0,
"Expecting Error because start time for new backup must be newer "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertRegex(
e.message,
"ERROR: Cannot create directory for older backup",
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
show_backup1 = self.show_pb(backup_dir1, 'node1')[1]
show_backup2 = self.show_pb(backup_dir2, 'node2')[0]
self.assertGreater(show_backup1['id'], show_backup2['id'])
# PAGE backup
startTime = int(time())
self.backup_node(
backup_dir1, 'node1', node1, backup_type="page",
options=['--stream', '--start-time', str(startTime)])
# make backup with start time definitelly earlier, than existing
try:
self.backup_node(
backup_dir2, 'node2', node2, backup_type="page",
options=['--stream', '--start-time', str(10000)])
self.assertEqual(
1, 0,
"Expecting Error because start time for new backup must be newer "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertRegex(
e.message,
"ERROR: Cannot create directory for older backup",
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
show_backup1 = self.show_pb(backup_dir1, 'node1')[2]
show_backup2 = self.show_pb(backup_dir2, 'node2')[0]
self.assertGreater(show_backup1['id'], show_backup2['id'])
# PTRACK backup
startTime = int(time())
if self.ptrack and node1.major_version > 11:
node1.safe_psql(
"postgres",
"create extension ptrack")
self.backup_node(
backup_dir1, 'node1', node1, backup_type="ptrack",
options=['--stream', '--start-time', str(startTime)])
if self.ptrack and node2.major_version > 11:
node2.safe_psql(
"postgres",
"create extension ptrack")
# make backup with start time definitelly earlier, than existing
try:
self.backup_node(
backup_dir2, 'node2', node2, backup_type="ptrack",
options=['--stream', '--start-time', str(10000)])
self.assertEqual(
1, 0,
"Expecting Error because start time for new backup must be newer "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertRegex(
e.message,
"ERROR: Cannot create directory for older backup",
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
# FULL backup
startTime = int(time())
self.backup_node(
backup_dir1, 'node1', node1, backup_type="full",
options=['--stream', '--start-time', str(startTime)])
self.backup_node(
backup_dir2, 'node2', node2, backup_type="full",
options=['--stream', '--start-time', str(startTime)])
show_backup1 = self.show_pb(backup_dir1, 'node1')[4]
show_backup2 = self.show_pb(backup_dir2, 'node2')[1]
self.assertEqual(show_backup1['id'], show_backup2['id'])
# Clean after yourself
self.del_test_dir(module_name, fname)

5
tests/helpers/ptrack_helpers.py
View File

@ -947,7 +947,7 @@ class ProbackupTest(object):
backup_type='full', datname=False, options=[],
asynchronous=False, gdb=False,
old_binary=False, return_id=True, no_remote=False,
env=None
env=None, startTime=None
):
if not node and not data_dir:
print('You must provide ether node or data_dir for backup')
@ -980,6 +980,9 @@ class ProbackupTest(object):
if not old_binary:
cmd_list += ['--no-sync']
if startTime:
cmd_list += ['--start-time', startTime]
return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env)
def checkdb_node(