diff --git a/src/backup.c b/src/backup.c index 96deb251..15ca8bf3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -706,7 +706,8 @@ do_backup_instance(void) if (!is_remote_backup) if (file->extra_dir_num) - dir_name = GetRelativePath(file->path, file->extradir); + dir_name = GetRelativePath(file->path, + parray_get(extra_dirs, file->extra_dir_num - 1)); else dir_name = GetRelativePath(file->path, instance_config.pgdata); else @@ -745,7 +746,8 @@ do_backup_instance(void) arg->from_root = instance_config.pgdata; arg->to_root = database_path; - arg->extra = extra_prefix; + arg->extra_prefix = extra_prefix; + arg->extra_dirs = extra_dirs; arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; arg->prev_start_lsn = prev_backup_start_lsn; @@ -787,9 +789,6 @@ do_backup_instance(void) parray_walk(prev_backup_filelist, pgFileFree); parray_free(prev_backup_filelist); } - /* clean extra directories list */ - if (extra_dirs) - free_dir_list(extra_dirs); /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -853,7 +852,12 @@ do_backup_instance(void) } /* Print the list of files to backup catalog */ - write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, NULL); + write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, + NULL, extra_dirs); + + /* clean extra directories list */ + if (extra_dirs) + free_dir_list(extra_dirs); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) @@ -2285,6 +2289,11 @@ backup_files(void *arg) if (S_ISREG(buf.st_mode)) { pgFile **prev_file = NULL; + char *extra_path = NULL; + + if (file->extra_dir_num) + extra_path = parray_get(arguments->extra_dirs, + file->extra_dir_num - 1); /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) @@ -2293,7 +2302,7 @@ backup_files(void *arg) pgFile key; relative = GetRelativePath(file->path, file->extra_dir_num ? - file->extradir:arguments->from_root); + extra_path : arguments->from_root); key.path = relative; key.extra_dir_num = file->extra_dir_num; @@ -2345,9 +2354,9 @@ backup_files(void *arg) /* Set file paths */ if (file->extra_dir_num) { - makeExtraDirPathByNum(extra_dst, arguments->extra, + makeExtraDirPathByNum(extra_dst, arguments->extra_prefix, file->extra_dir_num); - src = file->extradir; + src = extra_path; dst = extra_dst; } else diff --git a/src/catalog.c b/src/catalog.c index a4059c6a..fd257ea0 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -551,7 +551,7 @@ write_backup(pgBackup *backup) */ void write_backup_filelist(pgBackup *backup, parray *files, const char *root, - const char *extra_prefix) + const char *extra_prefix, parray *extra_list) { FILE *fp; char path[MAXPGPATH]; @@ -563,7 +563,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, elog(ERROR, "Cannot open file list \"%s\": %s", path, strerror(errno)); - print_file_list(fp, files, root, extra_prefix); + print_file_list(fp, files, root, extra_prefix, extra_list); if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || diff --git a/src/dir.c b/src/dir.c index 2a403069..b636e00f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -119,7 +119,7 @@ typedef struct TablespaceCreatedList static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list, + bool omit_symlink, parray *black_list, int extra_dir_num); static void list_data_directories(parray *files, const char *path, bool is_root, @@ -226,7 +226,6 @@ pgFileInit(const char *path) file->n_blocks = BLOCKNUM_INVALID; file->compress_alg = NOT_DEFINED_COMPRESS; file->extra_dir_num = 0; - file->extradir = NULL; return file; } @@ -347,7 +346,7 @@ pgFileComparePath(const void *f1, const void *f2) return strcmp(f1p->path, f2p->path); } -/* +/* * Compare two pgFile with their path and extra_dir_num * in ascending order of ASCII code. */ @@ -466,8 +465,6 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, elog(WARNING, "Skip \"%s\": unexpected file format", file->path); return; } - if (extra_dir_num) - file->extradir = pgut_strdup(file->path); if (add_root) parray_append(files, file); @@ -725,8 +722,6 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, } /* If it is extra dir, remember it */ - if (extra_dir_num) - file->extradir = parent->extradir; /* We add the directory anyway */ if (S_ISDIR(file->mode)) @@ -1229,7 +1224,7 @@ check_tablespace_mapping(pgBackup *backup) */ void print_file_list(FILE *out, const parray *files, const char *root, - const char *extra_prefix) + const char *extra_prefix, parray *extra_list) { size_t i; @@ -1243,7 +1238,11 @@ print_file_list(FILE *out, const parray *files, const char *root, if (root && strstr(path, root) == path) path = GetRelativePath(path, root); else if (file->extra_dir_num && !extra_prefix) - path = GetRelativePath(path, file->extradir); + { + Assert(extra_list); + path = GetRelativePath(path, parray_get(extra_list, + file->extra_dir_num - 1)); + } fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " @@ -1253,8 +1252,8 @@ print_file_list(FILE *out, const parray *files, const char *root, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg), file->extra_dir_num); - if (file->extradir) - fprintf(out, ",\"extradir\":\"%s\"", file->extradir); + //if (file->extradir) + // fprintf(out, ",\"extradir\":\"%s\"", file->extradir); if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); @@ -1439,7 +1438,6 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_ char filepath[MAXPGPATH]; char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; - char extradir_str[MAXPGPATH]; int64 write_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, @@ -1473,11 +1471,6 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_ file = pgFileInit(filepath); - if (extra_dir_num) - { - get_control_value(buf, "extradir", extradir_str, NULL, true); - file->extradir = pgut_strdup(extradir_str); - } file->write_size = (int64) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; diff --git a/src/merge.c b/src/merge.c index 0d320592..4b74d040 100644 --- a/src/merge.c +++ b/src/merge.c @@ -324,7 +324,10 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->stop_lsn = from_backup->stop_lsn; to_backup->recovery_time = from_backup->recovery_time; to_backup->recovery_xid = from_backup->recovery_xid; + + pfree(to_backup->extra_dir_str); to_backup->extra_dir_str = from_backup->extra_dir_str; + from_backup->extra_dir_str = NULL; /* For safe pgBackupFree() */ /* * If one of the backups isn't "stream" backup then the target backup become * non-stream backup too. @@ -351,7 +354,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path, - from_extra_prefix); + from_extra_prefix, NULL); write_backup(to_backup); delete_source_backup: @@ -370,7 +373,8 @@ delete_source_backup: pgFile *file = (pgFile *) parray_get(to_files, i); if (file->extra_dir_num && - backup_contains_extra(file->extradir, from_extra)) + backup_contains_extra(parray_get(to_extra, file->extra_dir_num - 1), + from_extra)) /* Dir already removed*/ continue; @@ -403,8 +407,6 @@ delete_source_backup: * Merging finished, now we can safely update ID of the destination backup. */ to_backup->start_time = from_backup->start_time; - if (from_backup->extra_dir_str) - to_backup->extra_dir_str = from_backup->extra_dir_str; write_backup(to_backup); /* Cleanup */ @@ -612,18 +614,20 @@ merge_files(void *arg) copy_pgcontrol_file(argument->from_root, argument->to_root, file); else if (file->extra_dir_num) { - char from_root[MAXPGPATH]; - char to_root[MAXPGPATH]; - int new_dir_num; + char from_root[MAXPGPATH]; + char to_root[MAXPGPATH]; + int new_dir_num; + char *file_extra_path = parray_get(argument->from_extra, + file->extra_dir_num - 1); Assert(argument->from_extra); - new_dir_num = get_extra_index(file->extradir, argument->from_extra); + new_dir_num = get_extra_index(file_extra_path, + argument->from_extra); makeExtraDirPathByNum(from_root, argument->from_extra_prefix, file->extra_dir_num); makeExtraDirPathByNum(to_root, argument->to_extra_prefix, new_dir_num); copy_file(from_root, to_root, file); - file->extra_dir_num = new_dir_num; } else copy_file(argument->from_root, argument->to_root, file); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index dc72481f..e472ffac 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -123,7 +123,6 @@ typedef struct pgFile bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; int extra_dir_num; /* Number of extra directory. 0 if not extra */ - char *extradir; /* File from extra directory */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ @@ -292,10 +291,11 @@ typedef struct { const char *from_root; const char *to_root; - const char *extra; + const char *extra_prefix; parray *files_list; parray *prev_filelist; + parray *extra_dirs; XLogRecPtr prev_start_lsn; PGconn *backup_conn; @@ -476,7 +476,8 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, - const char *root, const char *extra_prefix); + const char *root, const char *extra_prefix, + parray *extra_list); extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); @@ -511,7 +512,7 @@ extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root, - const char *extra_prefix); + const char *extra_prefix, parray *extra_list); extern parray *dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt); extern parray *make_extra_directory_list(const char *colon_separated_dirs); extern void free_dir_list(parray *list); diff --git a/src/restore.c b/src/restore.c index e75ef4d9..8d2b7c0c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -21,7 +21,8 @@ typedef struct { parray *files; pgBackup *backup; - parray *extra_dirs; + parray *req_extra_dirs; + parray *cur_extra_dirs; char *extra_prefix; /* @@ -424,7 +425,8 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) char extra_prefix[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; - parray *extra_dirs = NULL; + parray *requested_extra_dirs = NULL; + parray *current_extra_dirs = NULL; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; @@ -458,13 +460,16 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) if(extra_dir_str) { - extra_dirs = make_extra_directory_list(extra_dir_str); - for (int i = 0; i < parray_num(extra_dirs); i++) + requested_extra_dirs = make_extra_directory_list(extra_dir_str); + for (int i = 0; i < parray_num(requested_extra_dirs); i++) { - dir_create_dir(parray_get(extra_dirs, i), DIR_PERMISSION); + dir_create_dir(parray_get(requested_extra_dirs, i), DIR_PERMISSION); } } + if(backup->extra_dir_str) + current_extra_dirs = make_extra_directory_list(backup->extra_dir_str); + /* * Get list of files which need to be restored. */ @@ -489,15 +494,21 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) { char dirpath[MAXPGPATH]; char *dir_name; + char *extra_path; - if (backup_contains_extra(file->extradir, extra_dirs)) + if (!current_extra_dirs || + parray_num(current_extra_dirs) < file->extra_dir_num - 1) + elog(ERROR, "Inconsistent extra directory backup metadata"); + + extra_path = parray_get(current_extra_dirs, file->extra_dir_num - 1); + if (backup_contains_extra(extra_path, requested_extra_dirs)) { char container_dir[MAXPGPATH]; makeExtraDirPathByNum(container_dir, extra_prefix, file->extra_dir_num); dir_name = GetRelativePath(file->path, container_dir); elog(VERBOSE, "Create directory \"%s\"", dir_name); - join_path_components(dirpath, file->extradir, dir_name); + join_path_components(dirpath, extra_path, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); } } @@ -515,7 +526,8 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) arg->files = files; arg->backup = backup; - arg->extra_dirs = extra_dirs; + arg->req_extra_dirs = requested_extra_dirs; + arg->cur_extra_dirs = current_extra_dirs; arg->extra_prefix = extra_prefix; /* By default there are some error */ threads_args[i].ret = 1; @@ -678,8 +690,10 @@ restore_files(void *arg) copy_pgcontrol_file(from_root, instance_config.pgdata, file); else if (file->extra_dir_num) { - if (backup_contains_extra(file->extradir, arguments->extra_dirs)) - copy_file(arguments->extra_prefix, file->extradir, file); + char *extra_path = parray_get(arguments->cur_extra_dirs, + file->extra_dir_num - 1); + if (backup_contains_extra(extra_path, arguments->req_extra_dirs)) + copy_file(arguments->extra_prefix, extra_path, file); } else copy_file(from_root, instance_config.pgdata, file);