mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2025-01-23 11:45:36 +02:00
Restore extra directories listed only in last backup
This commit is contained in:
Sergey Cherkashin
parent
f41df91171
commit
3843f58933
45
src/backup.c
45
src/backup.c
@ -472,7 +472,6 @@ do_backup_instance(void)
|
||||
pthread_t *threads;
|
||||
backup_files_arg *threads_args;
|
||||
bool backup_isok = true;
|
||||
char *p;
|
||||
|
||||
pgBackup *prev_backup = NULL;
|
||||
parray *prev_backup_filelist = NULL;
|
||||
@ -482,19 +481,8 @@ do_backup_instance(void)
|
||||
pgFile *pg_control = NULL;
|
||||
|
||||
elog(LOG, "Database backup start");
|
||||
extra_dirs = parray_new();
|
||||
/* TODO: Add path validation */
|
||||
if(extradir)
|
||||
{
|
||||
p = strtok(extradir,":");
|
||||
while(p!=NULL)
|
||||
{
|
||||
char * dir = (char *)palloc(strlen(p) + 1);
|
||||
strcpy(dir,p);
|
||||
parray_append(extra_dirs, dir);
|
||||
p=strtok(NULL,":");
|
||||
}
|
||||
}
|
||||
if(current.extra_dir_str)
|
||||
extra_dirs = make_extra_directory_list(current.extra_dir_str);
|
||||
|
||||
/* Initialize size summary */
|
||||
current.data_bytes = 0;
|
||||
@ -633,7 +621,17 @@ do_backup_instance(void)
|
||||
if (is_remote_backup)
|
||||
get_remote_pgdata_filelist(backup_files_list);
|
||||
else
|
||||
dir_list_file(backup_files_list, pgdata, true, true, false, false);
|
||||
dir_list_file(backup_files_list, pgdata, true, true, false, 0);
|
||||
|
||||
/*
|
||||
* Append to backup list all files and directories
|
||||
* from extra directory option
|
||||
*/
|
||||
if (extra_dirs)
|
||||
for (i = 0; i < parray_num(extra_dirs); i++)
|
||||
/* Extra dirs numeration starts with 1. 0 value is not extra dir */
|
||||
dir_list_file(backup_files_list, parray_get(extra_dirs, i),
|
||||
true, true, false, i+1);
|
||||
|
||||
/*
|
||||
* Sort pathname ascending. It is necessary to create intermediate
|
||||
@ -651,12 +649,6 @@ do_backup_instance(void)
|
||||
/* Extract information about files in backup_list parsing their names:*/
|
||||
parse_backup_filelist_filenames(backup_files_list, pgdata);
|
||||
|
||||
/* Append to backup list all files dirictories from extra dirictory option */
|
||||
for (i = 0; i < parray_num(extra_dirs); i++)
|
||||
/* Extra dirs numeration starts with 1. 0 value is not extra dir */
|
||||
dir_list_file(backup_files_list, (char *) parray_get(extra_dirs, i),
|
||||
true, true, true, i+1);
|
||||
|
||||
if (current.backup_mode != BACKUP_MODE_FULL)
|
||||
{
|
||||
elog(LOG, "current_tli:%X", current.tli);
|
||||
@ -786,10 +778,7 @@ do_backup_instance(void)
|
||||
}
|
||||
/* clean extra directories list */
|
||||
if (extra_dirs)
|
||||
{
|
||||
parray_walk(extra_dirs, pfree);
|
||||
parray_free(extra_dirs);
|
||||
}
|
||||
free_dir_list(extra_dirs);
|
||||
|
||||
/* In case of backup from replica >= 9.6 we must fix minRecPoint,
|
||||
* First we must find pg_control in backup_files_list.
|
||||
@ -981,6 +970,10 @@ do_backup(time_t start_time)
|
||||
StrNCpy(current.program_version, PROGRAM_VERSION,
|
||||
sizeof(current.program_version));
|
||||
|
||||
/* Save list of extra directories */
|
||||
if(extradir)
|
||||
current.extra_dir_str = extradir;
|
||||
|
||||
/* Create backup directory and BACKUP_CONTROL_FILE */
|
||||
if (pgBackupCreateDir(¤t))
|
||||
elog(ERROR, "cannot create backup directory");
|
||||
@ -2246,7 +2239,7 @@ backup_files(void *arg)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
else
|
||||
else
|
||||
{
|
||||
const char *src;
|
||||
const char *dst;
|
||||
|
||||
@ -406,7 +406,26 @@ pgBackupCreateDir(pgBackup *backup)
|
||||
{
|
||||
int i;
|
||||
char path[MAXPGPATH];
|
||||
char *subdirs[] = { DATABASE_DIR, NULL };
|
||||
parray *subdirs = parray_new();
|
||||
char *temp;
|
||||
|
||||
temp = palloc(strlen(DATABASE_DIR) + 1);
|
||||
parray_append(subdirs, temp);
|
||||
|
||||
/* Add extra dirs containers */
|
||||
if (backup->extra_dir_str)
|
||||
{
|
||||
parray *extradirs_list = make_extra_directory_list(backup->extra_dir_str);
|
||||
for (int i = 0; i < parray_num(extradirs_list); i++)
|
||||
{
|
||||
/* 20 chars is enough to hold the extradir number in string. */
|
||||
temp = palloc(strlen(EXTRA_DIR) + 20);
|
||||
/* Numeration of extradirs starts with 1 */
|
||||
makeExtraDirPathByNum(temp, EXTRA_DIR, i+1);
|
||||
parray_append(subdirs, temp);
|
||||
}
|
||||
free_dir_list(extradirs_list);
|
||||
}
|
||||
|
||||
pgBackupGetPath(backup, path, lengthof(path), NULL);
|
||||
|
||||
@ -416,12 +435,13 @@ pgBackupCreateDir(pgBackup *backup)
|
||||
dir_create_dir(path, DIR_PERMISSION);
|
||||
|
||||
/* create directories for actual backup files */
|
||||
for (i = 0; subdirs[i]; i++)
|
||||
for (i = 0; i < parray_num(subdirs); i++)
|
||||
{
|
||||
pgBackupGetPath(backup, path, lengthof(path), subdirs[i]);
|
||||
pgBackupGetPath(backup, path, lengthof(path), parray_get(subdirs, i));
|
||||
dir_create_dir(path, DIR_PERMISSION);
|
||||
}
|
||||
|
||||
free_dir_list(subdirs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -494,6 +514,10 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
|
||||
/* print connection info except password */
|
||||
if (backup->primary_conninfo)
|
||||
fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo);
|
||||
|
||||
/* print extra directories list */
|
||||
if (backup->extra_dir_str)
|
||||
fprintf(out, "extra-directory = '%s'\n", backup->extra_dir_str);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -587,6 +611,7 @@ readBackupControlFile(const char *path)
|
||||
{'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT},
|
||||
{'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT},
|
||||
{'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT},
|
||||
{'s', 0, "extra-directory", &backup->extra_dir_str, SOURCE_FILE_STRICT},
|
||||
{0}
|
||||
};
|
||||
|
||||
@ -816,6 +841,7 @@ pgBackupInit(pgBackup *backup)
|
||||
backup->primary_conninfo = NULL;
|
||||
backup->program_version[0] = '\0';
|
||||
backup->server_version[0] = '\0';
|
||||
backup->extra_dir_str = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -830,6 +856,8 @@ pgBackupCopy(pgBackup *dst, pgBackup *src)
|
||||
|
||||
if (src->primary_conninfo)
|
||||
dst->primary_conninfo = pstrdup(src->primary_conninfo);
|
||||
if (src->extra_dir_str)
|
||||
dst->extra_dir_str = pstrdup(src->extra_dir_str);
|
||||
}
|
||||
|
||||
/* free pgBackup object */
|
||||
@ -839,6 +867,7 @@ pgBackupFree(void *backup)
|
||||
pgBackup *b = (pgBackup *) backup;
|
||||
|
||||
pfree(b->primary_conninfo);
|
||||
pfree(b->extra_dir_str);
|
||||
pfree(backup);
|
||||
}
|
||||
|
||||
|
||||
64
src/dir.c
64
src/dir.c
@ -115,8 +115,6 @@ typedef struct TablespaceCreatedList
|
||||
TablespaceCreatedListCell *tail;
|
||||
} TablespaceCreatedList;
|
||||
|
||||
static int BlackListCompare(const void *str1, const void *str2);
|
||||
|
||||
static bool dir_check_file(const char *root, pgFile *file);
|
||||
static void dir_list_file_internal(parray *files, const char *root,
|
||||
pgFile *parent, bool exclude,
|
||||
@ -385,7 +383,8 @@ pgFileCompareSize(const void *f1, const void *f2)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
/* Compare two strings */
|
||||
int
|
||||
BlackListCompare(const void *str1, const void *str2)
|
||||
{
|
||||
return strcmp(*(char **) str1, *(char **) str2);
|
||||
@ -446,15 +445,19 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
|
||||
elog(WARNING, "Skip \"%s\": unexpected file format", file->path);
|
||||
return;
|
||||
}
|
||||
if (add_root)
|
||||
if (extra_dir_num)
|
||||
{
|
||||
file->extradir = dirname(pgut_strdup(file->path));
|
||||
elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s",file->name, file->path, file->extradir);
|
||||
parray_append(files, file);
|
||||
file->extradir = pgut_strdup(file->path);
|
||||
elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s",
|
||||
file->name, file->path, file->extradir);
|
||||
}
|
||||
if (add_root)
|
||||
parray_append(files, file);
|
||||
|
||||
dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list,
|
||||
extra_dir_num);
|
||||
|
||||
free_dir_list(black_list);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -705,10 +708,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
|
||||
|
||||
/* If it is extra dir, remember it */
|
||||
if (extra_dir_num)
|
||||
{
|
||||
file->extradir = pgut_strdup(root);
|
||||
dirname(file->extradir);
|
||||
}
|
||||
file->extradir = parent->extradir;
|
||||
|
||||
/* We add the directory anyway */
|
||||
if (S_ISDIR(file->mode))
|
||||
@ -1550,3 +1550,45 @@ pgFileSize(const char *path)
|
||||
|
||||
return buf.st_size;
|
||||
}
|
||||
|
||||
/*
|
||||
* Construct parray containing extra directories paths
|
||||
* from string like /path1:/path2
|
||||
*/
|
||||
parray *
|
||||
make_extra_directory_list(const char *colon_separated_dirs)
|
||||
{
|
||||
char *p;
|
||||
parray *list = parray_new();
|
||||
char *tmp = palloc(strlen(colon_separated_dirs) + 1);
|
||||
|
||||
/* TODO: Add path validation */
|
||||
strcpy(tmp, colon_separated_dirs);
|
||||
p = strtok(tmp,":");
|
||||
while(p!=NULL)
|
||||
{
|
||||
char * dir = (char *)palloc(strlen(p) + 1);
|
||||
strcpy(dir,p);
|
||||
parray_append(list, dir);
|
||||
p=strtok(NULL,":");
|
||||
}
|
||||
pfree(tmp);
|
||||
parray_qsort(list, BlackListCompare);
|
||||
return list;
|
||||
}
|
||||
|
||||
/* Free memory of parray containing strings */
|
||||
void
|
||||
free_dir_list(parray *list)
|
||||
{
|
||||
parray_walk(list, pfree);
|
||||
parray_free(list);
|
||||
}
|
||||
|
||||
/* Append to string "pattern_path" int "dir_num" */
|
||||
void
|
||||
makeExtraDirPathByNum(char *ret_path, const char *pattern_path,
|
||||
const int dir_num)
|
||||
{
|
||||
sprintf(ret_path, "%s%d", pattern_path, dir_num);
|
||||
}
|
||||
|
||||
@ -251,6 +251,7 @@ struct pgBackup
|
||||
pgBackup *parent_backup_link;
|
||||
char *primary_conninfo; /* Connection parameters of the backup
|
||||
* in the format suitable for recovery.conf */
|
||||
char *extra_dir_str; /* List of extra directories, separated by ':' */
|
||||
};
|
||||
|
||||
/* Recovery target for restore and validate subcommands */
|
||||
@ -527,6 +528,10 @@ extern void check_tablespace_mapping(pgBackup *backup);
|
||||
|
||||
extern void print_file_list(FILE *out, const parray *files, const char *root);
|
||||
extern parray *dir_read_file_list(const char *root, const char *extra_path, const char *file_txt);
|
||||
extern parray *make_extra_directory_list(const char *colon_separated_dirs);
|
||||
extern void free_dir_list(parray *list);
|
||||
extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path,
|
||||
const int dir_num);
|
||||
|
||||
extern int dir_create_dir(const char *path, mode_t mode);
|
||||
extern bool dir_is_empty(const char *path);
|
||||
@ -544,6 +549,7 @@ extern int pgFileComparePathWithExtra(const void *f1, const void *f2);
|
||||
extern int pgFileComparePathDesc(const void *f1, const void *f2);
|
||||
extern int pgFileCompareLinked(const void *f1, const void *f2);
|
||||
extern int pgFileCompareSize(const void *f1, const void *f2);
|
||||
extern int BlackListCompare(const void *str1, const void *str2);
|
||||
|
||||
/* in data.c */
|
||||
extern bool backup_data_file(backup_files_arg* arguments,
|
||||
|
||||
@ -21,6 +21,7 @@ typedef struct
|
||||
{
|
||||
parray *files;
|
||||
pgBackup *backup;
|
||||
parray *extra_dirs;
|
||||
|
||||
/*
|
||||
* Return value from the thread.
|
||||
@ -29,14 +30,14 @@ typedef struct
|
||||
int ret;
|
||||
} restore_files_arg;
|
||||
|
||||
static void restore_backup(pgBackup *backup);
|
||||
static void restore_backup(pgBackup *backup, const char *extra_dir_str);
|
||||
static void create_recovery_conf(time_t backup_id,
|
||||
pgRecoveryTarget *rt,
|
||||
pgBackup *backup);
|
||||
static parray *read_timeline_history(TimeLineID targetTLI);
|
||||
static void *restore_files(void *arg);
|
||||
static void remove_deleted_files(pgBackup *backup);
|
||||
|
||||
static bool backup_contains_extra(const char *dir, parray *dirs_list);
|
||||
|
||||
/*
|
||||
* Entry point of pg_probackup RESTORE and VALIDATE subcommands.
|
||||
@ -387,7 +388,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
|
||||
elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn",
|
||||
base36enc(dest_backup->start_time), dest_backup->server_version);
|
||||
|
||||
restore_backup(backup);
|
||||
restore_backup(backup, dest_backup->extra_dir_str);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -414,7 +415,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
|
||||
* Restore one backup.
|
||||
*/
|
||||
void
|
||||
restore_backup(pgBackup *backup)
|
||||
restore_backup(pgBackup *backup, const char *extra_dir_str)
|
||||
{
|
||||
char timestamp[100];
|
||||
char this_backup_path[MAXPGPATH];
|
||||
@ -422,6 +423,7 @@ restore_backup(pgBackup *backup)
|
||||
char extra_path[MAXPGPATH];
|
||||
char list_path[MAXPGPATH];
|
||||
parray *files;
|
||||
parray *extra_dirs;
|
||||
int i;
|
||||
/* arrays with meta info for multi threaded backup */
|
||||
pthread_t *threads;
|
||||
@ -453,6 +455,15 @@ restore_backup(pgBackup *backup)
|
||||
pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL);
|
||||
create_data_directories(pgdata, this_backup_path, true);
|
||||
|
||||
if(extra_dir_str)
|
||||
{
|
||||
extra_dirs = make_extra_directory_list(extra_dir_str);
|
||||
for (int i = 0; i < parray_num(extra_dirs); i++)
|
||||
{
|
||||
dir_create_dir(parray_get(extra_dirs, i), DIR_PERMISSION);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Get list of files which need to be restored.
|
||||
*/
|
||||
@ -472,25 +483,23 @@ restore_backup(pgBackup *backup)
|
||||
{
|
||||
pgFile *file = (pgFile *) parray_get(files, i);
|
||||
|
||||
/* if the entry was a directory, create it in the backup */
|
||||
if (S_ISDIR(file->mode))
|
||||
/* if the entry was an extra directory, create it in the backup */
|
||||
if (file->extra_dir_num && S_ISDIR(file->mode))
|
||||
{
|
||||
char dirpath[MAXPGPATH];
|
||||
char *dir_name;
|
||||
|
||||
if (file->extra_dir_num)
|
||||
if (backup_contains_extra(file->extradir, extra_dirs))
|
||||
{
|
||||
char container_dir[MAXPGPATH];
|
||||
makeExtraDirPathByNum(container_dir, extra_path,
|
||||
file->extra_dir_num);
|
||||
dir_name = GetRelativePath(file->path, container_dir);
|
||||
elog(VERBOSE, "Create directory \"%s\" NAME %s",
|
||||
dir_name, file->name);
|
||||
join_path_components(dirpath, file->extradir, dir_name);
|
||||
elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name);
|
||||
dir_name = GetRelativePath(file->path, extra_path);
|
||||
dir_create_dir(dirpath, DIR_PERMISSION);
|
||||
}
|
||||
else
|
||||
{
|
||||
dir_name = GetRelativePath(file->path, database_path);
|
||||
elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name);
|
||||
join_path_components(dirpath, pgdata, dir_name);
|
||||
}
|
||||
dir_create_dir(dirpath, DIR_PERMISSION);
|
||||
}
|
||||
|
||||
/* setup threads */
|
||||
@ -506,6 +515,7 @@ restore_backup(pgBackup *backup)
|
||||
|
||||
arg->files = files;
|
||||
arg->backup = backup;
|
||||
arg->extra_dirs = extra_dirs;
|
||||
/* By default there are some error */
|
||||
threads_args[i].ret = 1;
|
||||
|
||||
@ -661,7 +671,10 @@ restore_files(void *arg)
|
||||
parse_program_version(arguments->backup->program_version));
|
||||
}
|
||||
else if (file->extra_dir_num)
|
||||
copy_file(from_root, file->extradir, file);
|
||||
{
|
||||
if (backup_contains_extra(file->extradir, arguments->extra_dirs))
|
||||
copy_file(from_root, file->extradir, file);
|
||||
}
|
||||
else
|
||||
copy_file(from_root, pgdata, file);
|
||||
|
||||
@ -1032,3 +1045,11 @@ parseRecoveryTargetOptions(const char *target_time,
|
||||
|
||||
return rt;
|
||||
}
|
||||
|
||||
/* Check if "dir" presents in "dirs_list" */
|
||||
static bool
|
||||
backup_contains_extra(const char *dir, parray *dirs_list)
|
||||
{
|
||||
void *temp = parray_bsearch(dirs_list, dir, BlackListCompare);
|
||||
return temp != NULL;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user