From 2220b49d6dd5ad1a54956f23571f51b05b917331 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 13 Dec 2018 21:02:27 +0300 Subject: [PATCH] Fix merge of extradirectories. Minor improvements. --- src/backup.c | 23 ++++++++++++----------- src/catalog.c | 1 + src/delete.c | 2 +- src/dir.c | 13 ++++--------- src/merge.c | 25 +++++++++++++++++++++---- src/pg_probackup.h | 2 +- src/restore.c | 27 ++++++++++++++------------- src/validate.c | 6 +++--- 8 files changed, 57 insertions(+), 42 deletions(-) diff --git a/src/backup.c b/src/backup.c index d1f3fa1b..96deb251 100644 --- a/src/backup.c +++ b/src/backup.c @@ -468,7 +468,7 @@ do_backup_instance(void) { int i; char database_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; /* Temp value. Used as template */ + char extra_prefix[MAXPGPATH]; /* Temp value. Used as template */ char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -579,7 +579,7 @@ do_backup_instance(void) pg_start_backup(label, smooth_checkpoint, ¤t); pgBackupGetPath(¤t, database_path, lengthof(database_path),DATABASE_DIR); - pgBackupGetPath(¤t, extra_path, lengthof(extra_path), EXTRA_DIR); + pgBackupGetPath(¤t, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); /* start stream replication */ if (stream_wal) @@ -712,12 +712,12 @@ do_backup_instance(void) else dir_name = file->path; - elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); + elog(VERBOSE, "Create directory \"%s\"", dir_name); if (file->extra_dir_num) { char temp[MAXPGPATH]; - snprintf(temp, MAXPGPATH, "%s%d", extra_path, file->extra_dir_num); + snprintf(temp, MAXPGPATH, "%s%d", extra_prefix, file->extra_dir_num); join_path_components(dirpath, temp, dir_name); } else @@ -745,7 +745,7 @@ do_backup_instance(void) arg->from_root = instance_config.pgdata; arg->to_root = database_path; - arg->extra = extra_path; + arg->extra = extra_prefix; arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; arg->prev_start_lsn = prev_backup_start_lsn; @@ -829,7 +829,7 @@ do_backup_instance(void) /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0); for (i = 0; i < parray_num(xlog_files_list); i++) { @@ -2040,7 +2040,7 @@ pg_stop_backup(pgBackup *backup) */ if (backup_files_list) { - file = pgFileNew(backup_label, true, false); + file = pgFileNew(backup_label, true, 0); calc_file_checksum(file); free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); @@ -2084,7 +2084,7 @@ pg_stop_backup(pgBackup *backup) if (backup_files_list) { - file = pgFileNew(tablespace_map, true, false); + file = pgFileNew(tablespace_map, true, 0); if (S_ISREG(file->mode)) calc_file_checksum(file); free(file->path); @@ -2331,6 +2331,7 @@ backup_files(void *arg) const char *src; const char *dst; bool skip = false; + char extra_dst[MAXPGPATH]; /* If non-data file has not changed since last backup... */ if (prev_file && file->exists_in_prev && @@ -2344,10 +2345,10 @@ backup_files(void *arg) /* Set file paths */ if (file->extra_dir_num) { - char temp[MAXPGPATH]; - sprintf(temp, "%s%d", arguments->extra, file->extra_dir_num); + makeExtraDirPathByNum(extra_dst, arguments->extra, + file->extra_dir_num); src = file->extradir; - dst = temp; + dst = extra_dst; } else { diff --git a/src/catalog.c b/src/catalog.c index 149c70b7..ed223557 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -854,6 +854,7 @@ void pgBackupCopy(pgBackup *dst, pgBackup *src) { pfree(dst->primary_conninfo); + pfree(dst->extra_dir_str); memcpy(dst, src, sizeof(pgBackup)); diff --git a/src/delete.c b/src/delete.c index b82f0dba..e01b613f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -283,7 +283,7 @@ delete_backup_files(pgBackup *backup) /* list files to be deleted */ files = parray_new(); pgBackupGetPath(backup, path, lengthof(path), NULL); - dir_list_file(files, path, false, true, true, false); + dir_list_file(files, path, false, true, true, 0); /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); diff --git a/src/dir.c b/src/dir.c index bc86c563..f166fa5e 100644 --- a/src/dir.c +++ b/src/dir.c @@ -15,7 +15,6 @@ #endif #include "catalog/pg_tablespace.h" -#include #include #include #include @@ -468,11 +467,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, return; } if (extra_dir_num) - { file->extradir = pgut_strdup(file->path); - elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s", - file->name, file->path, file->extradir); - } if (add_root) parray_append(files, file); @@ -835,7 +830,7 @@ list_data_directories(parray *files, const char *path, bool is_root, { pgFile *dir; - dir = pgFileNew(path, false, false); + dir = pgFileNew(path, false, 0); parray_append(files, dir); } @@ -1436,7 +1431,7 @@ bad_format: * If root is not NULL, path will be absolute path. */ parray * -dir_read_file_list(const char *root, const char *extra_path, const char *file_txt) +dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt) { FILE *fp; parray *files; @@ -1479,8 +1474,8 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx { char temp[MAXPGPATH]; - Assert(extra_path); - makeExtraDirPathByNum(temp, extra_path, extra_dir_num); + Assert(extra_prefix); + makeExtraDirPathByNum(temp, extra_prefix, extra_dir_num); join_path_components(filepath, temp, path); } else diff --git a/src/merge.c b/src/merge.c index 6ac8602e..0e9c75c5 100644 --- a/src/merge.c +++ b/src/merge.c @@ -170,9 +170,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) from_extra_prefix[MAXPGPATH], control_file[MAXPGPATH]; parray *files, - *to_files, - *to_extra, - *from_extra; + *to_files; + parray *to_extra = NULL, + *from_extra = NULL; pthread_t *threads = NULL; merge_files_arg *threads_args = NULL; int i; @@ -273,6 +273,23 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) { pgFile *file = (pgFile *) parray_get(files, i); + /* if the entry was an extra directory, create it in the backup */ + if (file->extra_dir_num && S_ISDIR(file->mode)) + { + char dirpath[MAXPGPATH]; + char *dir_name; + char old_container[MAXPGPATH]; + char new_container[MAXPGPATH]; + + makeExtraDirPathByNum(old_container, from_extra_prefix, + file->extra_dir_num); + makeExtraDirPathByNum(new_container, to_extra_prefix, + file->extra_dir_num); + dir_name = GetRelativePath(file->path, old_container); + elog(VERBOSE, "Create directory \"%s\"", dir_name); + join_path_components(dirpath, new_container, dir_name); + dir_create_dir(dirpath, DIR_PERMISSION); + } pg_atomic_init_flag(&file->lock); } @@ -316,6 +333,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->stop_lsn = from_backup->stop_lsn; to_backup->recovery_time = from_backup->recovery_time; to_backup->recovery_xid = from_backup->recovery_xid; + to_backup->extra_dir_str = from_backup->extra_dir_str; /* * If one of the backups isn't "stream" backup then the target backup become * non-stream backup too. @@ -625,7 +643,6 @@ reorder_extra_dirs(pgBackup *to_backup, parray *to_extra, parray *from_extra) { char extradir_template[MAXPGPATH]; - Assert(to_extra); pgBackupGetPath(to_backup, extradir_template, lengthof(extradir_template), EXTRA_DIR); for (int i = 0; i < parray_num(to_extra); i++) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3dfed3ac..1f7ef002 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -513,7 +513,7 @@ extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root, const char *extra_prefix); -extern parray *dir_read_file_list(const char *root, const char *extra_path, const char *file_txt); +extern parray *dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt); extern parray *make_extra_directory_list(const char *colon_separated_dirs); extern void free_dir_list(parray *list); extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path, diff --git a/src/restore.c b/src/restore.c index 41bf0410..1a9f76d1 100644 --- a/src/restore.c +++ b/src/restore.c @@ -22,6 +22,7 @@ typedef struct parray *files; pgBackup *backup; parray *extra_dirs; + char *extra_prefix; /* * Return value from the thread. @@ -420,10 +421,10 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) char timestamp[100]; char this_backup_path[MAXPGPATH]; char database_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; + char extra_prefix[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; - parray *extra_dirs; + parray *extra_dirs = NULL; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; @@ -468,15 +469,15 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) * Get list of files which need to be restored. */ pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); - pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); + pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); - files = dir_read_file_list(database_path, extra_path, list_path); + files = dir_read_file_list(database_path, extra_prefix, list_path); /* Restore directories in do_backup_instance way */ parray_qsort(files, pgFileComparePath); /* - * Make directories before backup + * Make extra directories before restore * and setup threads at the same time */ for (i = 0; i < parray_num(files); i++) @@ -492,11 +493,10 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) if (backup_contains_extra(file->extradir, extra_dirs)) { char container_dir[MAXPGPATH]; - makeExtraDirPathByNum(container_dir, extra_path, + makeExtraDirPathByNum(container_dir, extra_prefix, file->extra_dir_num); dir_name = GetRelativePath(file->path, container_dir); - elog(VERBOSE, "Create directory \"%s\" NAME %s", - dir_name, file->name); + elog(VERBOSE, "Create directory \"%s\"", dir_name); join_path_components(dirpath, file->extradir, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); } @@ -516,6 +516,7 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) arg->files = files; arg->backup = backup; arg->extra_dirs = extra_dirs; + arg->extra_prefix = extra_prefix; /* By default there are some error */ threads_args[i].ret = 1; @@ -558,18 +559,18 @@ remove_deleted_files(pgBackup *backup) parray *files; parray *files_restored; char filelist_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; + char extra_prefix[MAXPGPATH]; int i; pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); - pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); + pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(instance_config.pgdata, extra_path, filelist_path); + files = dir_read_file_list(instance_config.pgdata, extra_prefix, filelist_path); parray_qsort(files, pgFileComparePathDesc); /* Get list of files actually existing in target database */ files_restored = parray_new(); - dir_list_file(files_restored, instance_config.pgdata, true, true, false, false); + dir_list_file(files_restored, instance_config.pgdata, true, true, false, 0); /* To delete from leaf, sort in reversed order */ parray_qsort(files_restored, pgFileComparePathDesc); @@ -678,7 +679,7 @@ restore_files(void *arg) else if (file->extra_dir_num) { if (backup_contains_extra(file->extradir, arguments->extra_dirs)) - copy_file(from_root, file->extradir, file); + copy_file(arguments->extra_prefix, file->extradir, file); } else copy_file(from_root, instance_config.pgdata, file); diff --git a/src/validate.c b/src/validate.c index 1b893ec4..a241d7e5 100644 --- a/src/validate.c +++ b/src/validate.c @@ -43,7 +43,7 @@ void pgBackupValidate(pgBackup *backup) { char base_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; + char extra_prefix[MAXPGPATH]; char path[MAXPGPATH]; parray *files; bool corrupted = false; @@ -83,9 +83,9 @@ pgBackupValidate(pgBackup *backup) elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); - pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); + pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - files = dir_read_file_list(base_path, extra_path, path); + files = dir_read_file_list(base_path, extra_prefix, path); /* setup threads */ for (i = 0; i < parray_num(files); i++)