From 863486be7eab7f20e3ba14b58aa93c1d0cdb2b76 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 27 Nov 2018 16:28:30 +0300 Subject: [PATCH] Remove the limit on the number of extra directories. Save extra directories separately to eliminate name collisions. --- src/backup.c | 45 +++++++++++++++++++++++++++------------------ src/catalog.c | 2 +- src/dir.c | 38 ++++++++++++++++++++++++-------------- src/pg_probackup.h | 5 +++-- 4 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/backup.c b/src/backup.c index 9545d713..9f2f1b6f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -463,8 +463,7 @@ do_backup_instance(void) { int i; char database_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; - char *extradirs[] = { NULL, NULL, NULL, NULL }; + char extra_path[MAXPGPATH]; /* Temp value. Used as template */ char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -478,23 +477,22 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; parray *backup_list = NULL; + parray *extra_dirs = NULL; pgFile *pg_control = NULL; elog(LOG, "Database backup start"); - i = 0; + extra_dirs = parray_new(); + /* TODO: Add path validation */ if(extradir) { p = strtok(extradir,":"); while(p!=NULL) { - extradirs[i] = (char *)palloc(strlen(p) + 1); - strcpy(extradirs[i],p); - elog(WARNING,"%s",extradirs[i]); - i++; + char * dir = (char *)palloc(strlen(p) + 1); + strcpy(dir,p); + parray_append(extra_dirs, dir); p=strtok(NULL,":"); - if (i==3) - break; } } @@ -654,11 +652,10 @@ do_backup_instance(void) parse_backup_filelist_filenames(backup_files_list, pgdata); /* Append to backup list all files dirictories from extra dirictory option */ - for (i = 0; extradirs[i]; i++) - { - elog(WARNING,"%s",extradirs[i]); - dir_list_file(backup_files_list, extradirs[i], true, true, true, true); - } + for (i = 0; i < parray_num(extra_dirs); i++) + /* Extra dirs numeration starts with 1. 0 value is not extra dir */ + dir_list_file(backup_files_list, (char *) parray_get(extra_dirs, i), + true, true, true, i+1); if (current.backup_mode != BACKUP_MODE_FULL) { @@ -714,8 +711,12 @@ do_backup_instance(void) elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); - if (file->is_extra) - join_path_components(dirpath, extra_path, dir_name); + if (file->extra_dir_num) + { + char temp[MAXPGPATH]; + sprintf(temp, "%s%d", extra_path, file->extra_dir_num); + join_path_components(dirpath, temp, dir_name); + } else join_path_components(dirpath, database_path, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); @@ -783,6 +784,12 @@ do_backup_instance(void) parray_walk(prev_backup_filelist, pgFileFree); parray_free(prev_backup_filelist); } + /* clean extra directories list */ + if (extra_dirs) + { + parray_walk(extra_dirs, pfree); + parray_free(extra_dirs); + } /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -2237,10 +2244,12 @@ backup_files(void *arg) continue; } } - else if (file->is_extra) + else if (file->extra_dir_num) { + char temp[MAXPGPATH]; + sprintf(temp, "%s%d", arguments->extra, file->extra_dir_num); if (!copy_file(file->extradir, - arguments->extra, + temp, file)) { file->write_size = BYTES_INVALID; diff --git a/src/catalog.c b/src/catalog.c index fd75ef09..788c33f7 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -406,7 +406,7 @@ pgBackupCreateDir(pgBackup *backup) { int i; char path[MAXPGPATH]; - char *subdirs[] = { DATABASE_DIR, EXTRA_DIR, NULL }; + char *subdirs[] = { DATABASE_DIR, NULL }; pgBackupGetPath(backup, path, lengthof(path), NULL); diff --git a/src/dir.c b/src/dir.c index 7c341a28..8b1c193f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -120,7 +120,8 @@ static int BlackListCompare(const void *str1, const void *str2); static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list, bool is_extra); + bool omit_symlink, parray *black_list, + int extra_dir_num); static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude); @@ -156,7 +157,7 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, bool omit_symlink, bool is_extra) +pgFileNew(const char *path, bool omit_symlink, int extra_dir_num) { struct stat st; pgFile *file; @@ -174,7 +175,8 @@ pgFileNew(const char *path, bool omit_symlink, bool is_extra) file = pgFileInit(path); file->size = st.st_size; file->mode = st.st_mode; - file->is_extra = is_extra; + file->is_extra = extra_dir_num > 0; + file->extra_dir_num = extra_dir_num; file->extradir = NULL; return file; @@ -374,7 +376,7 @@ BlackListCompare(const void *str1, const void *str2) */ void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, - bool add_root, bool is_extra) + bool add_root, int extra_dir_num) { pgFile *file; parray *black_list = NULL; @@ -411,7 +413,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, false, is_extra); + file = pgFileNew(root, false, extra_dir_num); if (file == NULL) return; @@ -427,7 +429,8 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(files, file); } - dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, is_extra); + dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, + extra_dir_num); } /* @@ -615,7 +618,7 @@ dir_check_file(const char *root, pgFile *file) */ static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, - bool exclude, bool omit_symlink, parray *black_list, bool is_extra) + bool exclude, bool omit_symlink, parray *black_list, int extra_dir_num) { DIR *dir; struct dirent *dent; @@ -644,7 +647,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, join_path_components(child, parent->path, dent->d_name); - file = pgFileNew(child, omit_symlink, is_extra); + file = pgFileNew(child, omit_symlink, extra_dir_num); if (file == NULL) continue; @@ -677,7 +680,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, } /* If it is extra dir, remember it */ - if (is_extra) + if (extra_dir_num) { file->extradir = pgut_strdup(root); dirname(file->extradir); @@ -705,7 +708,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, root, file, exclude, omit_symlink, - black_list, is_extra); + black_list, extra_dir_num); } if (errno && errno != ENOENT) @@ -1201,11 +1204,12 @@ print_file_list(FILE *out, const parray *files, const char *root) fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"is_extra\":\"%u\"", + "\"compress_alg\":\"%s\", \"is_extra\":\"%u\"" + ", \"extra_dir_num\":\"%u\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg), - file->is_extra ? 1 : 0); + file->is_extra ? 1 : 0, file->extra_dir_num); if (file->extradir) fprintf(out, ",\"extradir\":\"%s\"", file->extradir); @@ -1399,6 +1403,7 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx is_datafile, is_cfs, is_extra, + extra_dir_num, crc, segno, n_blocks; @@ -1412,10 +1417,15 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx get_control_value(buf, "crc", NULL, &crc, true); get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); get_control_value(buf, "is_extra", NULL, &is_extra, false); + get_control_value(buf, "extra_dir_num", NULL, &extra_dir_num, false); if (root) - if (is_extra) - join_path_components(filepath, extra_path, path); + if (extra_dir_num) + { + char temp[MAXPGPATH]; + sprintf(temp, "%s%ld", extra_path, extra_dir_num); + join_path_components(filepath, temp, path); + } else join_path_components(filepath, root, path); else diff --git a/src/pg_probackup.h b/src/pg_probackup.h index dae77e72..83cbbb2c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -117,6 +117,7 @@ typedef struct pgFile bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; bool is_extra; + bool extra_dir_num; /* Number of extra directory. 0 if not extra */ char *extradir; /* File from extra directory */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ @@ -516,7 +517,7 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool omit_symlink, bool add_root, bool is_extra); + bool omit_symlink, bool add_root, int extra_dir_num); extern void create_data_directories(const char *data_dir, const char *backup_dir, bool extract_tablespaces); @@ -534,7 +535,7 @@ extern bool dir_is_empty(const char *path); extern bool fileExists(const char *path); extern size_t pgFileSize(const char *path); -extern pgFile *pgFileNew(const char *path, bool omit_symlink, bool is_extra); +extern pgFile *pgFileNew(const char *path, bool omit_symlink, int extra_dir_num); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file);