From 3843f5893368fd1a9c45f4f1135d9fc5808882d5 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Mon, 3 Dec 2018 20:07:59 +0300 Subject: [PATCH] Restore extra directories listed only in last backup --- src/backup.c | 45 ++++++++++++++------------------ src/catalog.c | 35 ++++++++++++++++++++++--- src/dir.c | 64 ++++++++++++++++++++++++++++++++++++++-------- src/pg_probackup.h | 6 +++++ src/restore.c | 55 +++++++++++++++++++++++++++------------ 5 files changed, 148 insertions(+), 57 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5b911689..91813ebb 100644 --- a/src/backup.c +++ b/src/backup.c @@ -472,7 +472,6 @@ do_backup_instance(void) pthread_t *threads; backup_files_arg *threads_args; bool backup_isok = true; - char *p; pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; @@ -482,19 +481,8 @@ do_backup_instance(void) pgFile *pg_control = NULL; elog(LOG, "Database backup start"); - extra_dirs = parray_new(); - /* TODO: Add path validation */ - if(extradir) - { - p = strtok(extradir,":"); - while(p!=NULL) - { - char * dir = (char *)palloc(strlen(p) + 1); - strcpy(dir,p); - parray_append(extra_dirs, dir); - p=strtok(NULL,":"); - } - } + if(current.extra_dir_str) + extra_dirs = make_extra_directory_list(current.extra_dir_str); /* Initialize size summary */ current.data_bytes = 0; @@ -633,7 +621,17 @@ do_backup_instance(void) if (is_remote_backup) get_remote_pgdata_filelist(backup_files_list); else - dir_list_file(backup_files_list, pgdata, true, true, false, false); + dir_list_file(backup_files_list, pgdata, true, true, false, 0); + + /* + * Append to backup list all files and directories + * from extra directory option + */ + if (extra_dirs) + for (i = 0; i < parray_num(extra_dirs); i++) + /* Extra dirs numeration starts with 1. 0 value is not extra dir */ + dir_list_file(backup_files_list, parray_get(extra_dirs, i), + true, true, false, i+1); /* * Sort pathname ascending. It is necessary to create intermediate @@ -651,12 +649,6 @@ do_backup_instance(void) /* Extract information about files in backup_list parsing their names:*/ parse_backup_filelist_filenames(backup_files_list, pgdata); - /* Append to backup list all files dirictories from extra dirictory option */ - for (i = 0; i < parray_num(extra_dirs); i++) - /* Extra dirs numeration starts with 1. 0 value is not extra dir */ - dir_list_file(backup_files_list, (char *) parray_get(extra_dirs, i), - true, true, true, i+1); - if (current.backup_mode != BACKUP_MODE_FULL) { elog(LOG, "current_tli:%X", current.tli); @@ -786,10 +778,7 @@ do_backup_instance(void) } /* clean extra directories list */ if (extra_dirs) - { - parray_walk(extra_dirs, pfree); - parray_free(extra_dirs); - } + free_dir_list(extra_dirs); /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -981,6 +970,10 @@ do_backup(time_t start_time) StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); + /* Save list of extra directories */ + if(extradir) + current.extra_dir_str = extradir; + /* Create backup directory and BACKUP_CONTROL_FILE */ if (pgBackupCreateDir(¤t)) elog(ERROR, "cannot create backup directory"); @@ -2246,7 +2239,7 @@ backup_files(void *arg) continue; } } - else + else { const char *src; const char *dst; diff --git a/src/catalog.c b/src/catalog.c index 788c33f7..8de7bb56 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -406,7 +406,26 @@ pgBackupCreateDir(pgBackup *backup) { int i; char path[MAXPGPATH]; - char *subdirs[] = { DATABASE_DIR, NULL }; + parray *subdirs = parray_new(); + char *temp; + + temp = palloc(strlen(DATABASE_DIR) + 1); + parray_append(subdirs, temp); + + /* Add extra dirs containers */ + if (backup->extra_dir_str) + { + parray *extradirs_list = make_extra_directory_list(backup->extra_dir_str); + for (int i = 0; i < parray_num(extradirs_list); i++) + { + /* 20 chars is enough to hold the extradir number in string. */ + temp = palloc(strlen(EXTRA_DIR) + 20); + /* Numeration of extradirs starts with 1 */ + makeExtraDirPathByNum(temp, EXTRA_DIR, i+1); + parray_append(subdirs, temp); + } + free_dir_list(extradirs_list); + } pgBackupGetPath(backup, path, lengthof(path), NULL); @@ -416,12 +435,13 @@ pgBackupCreateDir(pgBackup *backup) dir_create_dir(path, DIR_PERMISSION); /* create directories for actual backup files */ - for (i = 0; subdirs[i]; i++) + for (i = 0; i < parray_num(subdirs); i++) { - pgBackupGetPath(backup, path, lengthof(path), subdirs[i]); + pgBackupGetPath(backup, path, lengthof(path), parray_get(subdirs, i)); dir_create_dir(path, DIR_PERMISSION); } + free_dir_list(subdirs); return 0; } @@ -494,6 +514,10 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) /* print connection info except password */ if (backup->primary_conninfo) fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); + + /* print extra directories list */ + if (backup->extra_dir_str) + fprintf(out, "extra-directory = '%s'\n", backup->extra_dir_str); } /* @@ -587,6 +611,7 @@ readBackupControlFile(const char *path) {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, + {'s', 0, "extra-directory", &backup->extra_dir_str, SOURCE_FILE_STRICT}, {0} }; @@ -816,6 +841,7 @@ pgBackupInit(pgBackup *backup) backup->primary_conninfo = NULL; backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; + backup->extra_dir_str = NULL; } /* @@ -830,6 +856,8 @@ pgBackupCopy(pgBackup *dst, pgBackup *src) if (src->primary_conninfo) dst->primary_conninfo = pstrdup(src->primary_conninfo); + if (src->extra_dir_str) + dst->extra_dir_str = pstrdup(src->extra_dir_str); } /* free pgBackup object */ @@ -839,6 +867,7 @@ pgBackupFree(void *backup) pgBackup *b = (pgBackup *) backup; pfree(b->primary_conninfo); + pfree(b->extra_dir_str); pfree(backup); } diff --git a/src/dir.c b/src/dir.c index fe0804ed..98cc518d 100644 --- a/src/dir.c +++ b/src/dir.c @@ -115,8 +115,6 @@ typedef struct TablespaceCreatedList TablespaceCreatedListCell *tail; } TablespaceCreatedList; -static int BlackListCompare(const void *str1, const void *str2); - static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, @@ -385,7 +383,8 @@ pgFileCompareSize(const void *f1, const void *f2) return 0; } -static int +/* Compare two strings */ +int BlackListCompare(const void *str1, const void *str2) { return strcmp(*(char **) str1, *(char **) str2); @@ -446,15 +445,19 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, elog(WARNING, "Skip \"%s\": unexpected file format", file->path); return; } - if (add_root) + if (extra_dir_num) { - file->extradir = dirname(pgut_strdup(file->path)); - elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s",file->name, file->path, file->extradir); - parray_append(files, file); + file->extradir = pgut_strdup(file->path); + elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s", + file->name, file->path, file->extradir); } + if (add_root) + parray_append(files, file); dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, extra_dir_num); + + free_dir_list(black_list); } /* @@ -705,10 +708,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, /* If it is extra dir, remember it */ if (extra_dir_num) - { - file->extradir = pgut_strdup(root); - dirname(file->extradir); - } + file->extradir = parent->extradir; /* We add the directory anyway */ if (S_ISDIR(file->mode)) @@ -1550,3 +1550,45 @@ pgFileSize(const char *path) return buf.st_size; } + +/* + * Construct parray containing extra directories paths + * from string like /path1:/path2 + */ +parray * +make_extra_directory_list(const char *colon_separated_dirs) +{ + char *p; + parray *list = parray_new(); + char *tmp = palloc(strlen(colon_separated_dirs) + 1); + + /* TODO: Add path validation */ + strcpy(tmp, colon_separated_dirs); + p = strtok(tmp,":"); + while(p!=NULL) + { + char * dir = (char *)palloc(strlen(p) + 1); + strcpy(dir,p); + parray_append(list, dir); + p=strtok(NULL,":"); + } + pfree(tmp); + parray_qsort(list, BlackListCompare); + return list; +} + +/* Free memory of parray containing strings */ +void +free_dir_list(parray *list) +{ + parray_walk(list, pfree); + parray_free(list); +} + +/* Append to string "pattern_path" int "dir_num" */ +void +makeExtraDirPathByNum(char *ret_path, const char *pattern_path, + const int dir_num) +{ + sprintf(ret_path, "%s%d", pattern_path, dir_num); +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2cae08b8..d0230f45 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -251,6 +251,7 @@ struct pgBackup pgBackup *parent_backup_link; char *primary_conninfo; /* Connection parameters of the backup * in the format suitable for recovery.conf */ + char *extra_dir_str; /* List of extra directories, separated by ':' */ }; /* Recovery target for restore and validate subcommands */ @@ -527,6 +528,10 @@ extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root); extern parray *dir_read_file_list(const char *root, const char *extra_path, const char *file_txt); +extern parray *make_extra_directory_list(const char *colon_separated_dirs); +extern void free_dir_list(parray *list); +extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path, + const int dir_num); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); @@ -544,6 +549,7 @@ extern int pgFileComparePathWithExtra(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); +extern int BlackListCompare(const void *str1, const void *str2); /* in data.c */ extern bool backup_data_file(backup_files_arg* arguments, diff --git a/src/restore.c b/src/restore.c index de1ba4e5..9c176dae 100644 --- a/src/restore.c +++ b/src/restore.c @@ -21,6 +21,7 @@ typedef struct { parray *files; pgBackup *backup; + parray *extra_dirs; /* * Return value from the thread. @@ -29,14 +30,14 @@ typedef struct int ret; } restore_files_arg; -static void restore_backup(pgBackup *backup); +static void restore_backup(pgBackup *backup, const char *extra_dir_str); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); - +static bool backup_contains_extra(const char *dir, parray *dirs_list); /* * Entry point of pg_probackup RESTORE and VALIDATE subcommands. @@ -387,7 +388,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", base36enc(dest_backup->start_time), dest_backup->server_version); - restore_backup(backup); + restore_backup(backup, dest_backup->extra_dir_str); } /* @@ -414,7 +415,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Restore one backup. */ void -restore_backup(pgBackup *backup) +restore_backup(pgBackup *backup, const char *extra_dir_str) { char timestamp[100]; char this_backup_path[MAXPGPATH]; @@ -422,6 +423,7 @@ restore_backup(pgBackup *backup) char extra_path[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; + parray *extra_dirs; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; @@ -453,6 +455,15 @@ restore_backup(pgBackup *backup) pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); create_data_directories(pgdata, this_backup_path, true); + if(extra_dir_str) + { + extra_dirs = make_extra_directory_list(extra_dir_str); + for (int i = 0; i < parray_num(extra_dirs); i++) + { + dir_create_dir(parray_get(extra_dirs, i), DIR_PERMISSION); + } + } + /* * Get list of files which need to be restored. */ @@ -472,25 +483,23 @@ restore_backup(pgBackup *backup) { pgFile *file = (pgFile *) parray_get(files, i); - /* if the entry was a directory, create it in the backup */ - if (S_ISDIR(file->mode)) + /* if the entry was an extra directory, create it in the backup */ + if (file->extra_dir_num && S_ISDIR(file->mode)) { char dirpath[MAXPGPATH]; char *dir_name; - if (file->extra_dir_num) + if (backup_contains_extra(file->extradir, extra_dirs)) { + char container_dir[MAXPGPATH]; + makeExtraDirPathByNum(container_dir, extra_path, + file->extra_dir_num); + dir_name = GetRelativePath(file->path, container_dir); + elog(VERBOSE, "Create directory \"%s\" NAME %s", + dir_name, file->name); join_path_components(dirpath, file->extradir, dir_name); - elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); - dir_name = GetRelativePath(file->path, extra_path); + dir_create_dir(dirpath, DIR_PERMISSION); } - else - { - dir_name = GetRelativePath(file->path, database_path); - elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); - join_path_components(dirpath, pgdata, dir_name); - } - dir_create_dir(dirpath, DIR_PERMISSION); } /* setup threads */ @@ -506,6 +515,7 @@ restore_backup(pgBackup *backup) arg->files = files; arg->backup = backup; + arg->extra_dirs = extra_dirs; /* By default there are some error */ threads_args[i].ret = 1; @@ -661,7 +671,10 @@ restore_files(void *arg) parse_program_version(arguments->backup->program_version)); } else if (file->extra_dir_num) - copy_file(from_root, file->extradir, file); + { + if (backup_contains_extra(file->extradir, arguments->extra_dirs)) + copy_file(from_root, file->extradir, file); + } else copy_file(from_root, pgdata, file); @@ -1032,3 +1045,11 @@ parseRecoveryTargetOptions(const char *target_time, return rt; } + +/* Check if "dir" presents in "dirs_list" */ +static bool +backup_contains_extra(const char *dir, parray *dirs_list) +{ + void *temp = parray_bsearch(dirs_list, dir, BlackListCompare); + return temp != NULL; +}