db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-20 11:34:51 +02:00
Code Issues Releases Activity

Rename feature from "extra directories" to "external directories"

Browse Source
This commit is contained in:
Sergey Cherkashin 2019-02-12 17:19:18 +03:00
parent a12fe744f9
commit dbefa9e183
11 changed files with 289 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
src/backup.c
View File

@ -468,7 +468,7 @@ do_backup_instance(void)
{
int i;
char database_path[MAXPGPATH];
char extra_prefix[MAXPGPATH]; /* Temp value. Used as template */
char external_prefix[MAXPGPATH]; /* Temp value. Used as template */
char dst_backup_path[MAXPGPATH];
char label[1024];
XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr;
@ -481,13 +481,13 @@ do_backup_instance(void)
pgBackup *prev_backup = NULL;
parray *prev_backup_filelist = NULL;
parray *backup_list = NULL;
parray *extra_dirs = NULL;
parray *external_dirs = NULL;
pgFile *pg_control = NULL;
elog(LOG, "Database backup start");
if(current.extra_dir_str)
extra_dirs = make_extra_directory_list(current.extra_dir_str);
if(current.external_dir_str)
external_dirs = make_external_directory_list(current.external_dir_str);
/* Initialize size summary */
current.data_bytes = 0;
@ -579,7 +579,8 @@ do_backup_instance(void)
pg_start_backup(label, smooth_checkpoint, &current);
pgBackupGetPath(&current, database_path, lengthof(database_path),DATABASE_DIR);
pgBackupGetPath(&current, extra_prefix, lengthof(extra_prefix), EXTRA_DIR);
pgBackupGetPath(&current, external_prefix, lengthof(external_prefix),
EXTERNAL_DIR);
/* start stream replication */
if (stream_wal)
@ -636,12 +637,13 @@ do_backup_instance(void)
/*
* Append to backup list all files and directories
* from extra directory option
* from external directory option
*/
if (extra_dirs)
for (i = 0; i < parray_num(extra_dirs); i++)
/* Extra dirs numeration starts with 1. 0 value is not extra dir */
dir_list_file(backup_files_list, parray_get(extra_dirs, i),
if (external_dirs)
for (i = 0; i < parray_num(external_dirs); i++)
/* External dirs numeration starts with 1.
* 0 value is not external dir */
dir_list_file(backup_files_list, parray_get(external_dirs, i),
false, true, false, i+1);
/*
@ -705,9 +707,10 @@ do_backup_instance(void)
char *dir_name;
if (!is_remote_backup)
if (file->extra_dir_num)
if (file->external_dir_num)
dir_name = GetRelativePath(file->path,
parray_get(extra_dirs, file->extra_dir_num - 1));
parray_get(external_dirs,
file->external_dir_num - 1));
else
dir_name = GetRelativePath(file->path, instance_config.pgdata);
else
@ -715,10 +718,11 @@ do_backup_instance(void)
elog(VERBOSE, "Create directory \"%s\"", dir_name);
if (file->extra_dir_num)
if (file->external_dir_num)
{
char temp[MAXPGPATH];
snprintf(temp, MAXPGPATH, "%s%d", extra_prefix, file->extra_dir_num);
snprintf(temp, MAXPGPATH, "%s%d", external_prefix,
file->external_dir_num);
join_path_components(dirpath, temp, dir_name);
}
else
@ -734,7 +738,7 @@ do_backup_instance(void)
parray_qsort(backup_files_list, pgFileCompareSize);
/* Sort the array for binary search */
if (prev_backup_filelist)
parray_qsort(prev_backup_filelist, pgFileComparePathWithExtra);
parray_qsort(prev_backup_filelist, pgFileComparePathWithExternal);
/* init thread args with own file lists */
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
@ -746,8 +750,8 @@ do_backup_instance(void)
arg->from_root = instance_config.pgdata;
arg->to_root = database_path;
arg->extra_prefix = extra_prefix;
arg->extra_dirs = extra_dirs;
arg->external_prefix = external_prefix;
arg->external_dirs = external_dirs;
arg->files_list = backup_files_list;
arg->prev_filelist = prev_backup_filelist;
arg->prev_start_lsn = prev_backup_start_lsn;
@ -853,11 +857,11 @@ do_backup_instance(void)
/* Print the list of files to backup catalog */
write_backup_filelist(&current, backup_files_list, instance_config.pgdata,
NULL, extra_dirs);
NULL, external_dirs);
/* clean extra directories list */
if (extra_dirs)
free_dir_list(extra_dirs);
/* clean external directories list */
if (external_dirs)
free_dir_list(external_dirs);
/* Compute summary of size of regular files in the backup */
for (i = 0; i < parray_num(backup_files_list); i++)
@ -990,11 +994,11 @@ do_backup(time_t start_time)
StrNCpy(current.program_version, PROGRAM_VERSION,
sizeof(current.program_version));
/* Save list of extra directories */
if (instance_config.extra_dir_str &&
pg_strcasecmp(instance_config.extra_dir_str, "none") != 0)
/* Save list of external directories */
if (instance_config.external_dir_str &&
pg_strcasecmp(instance_config.external_dir_str, "none") != 0)
{
current.extra_dir_str = instance_config.extra_dir_str;
current.external_dir_str = instance_config.external_dir_str;
}
/* Create backup directory and BACKUP_CONTROL_FILE */
@ -2292,11 +2296,11 @@ backup_files(void *arg)
if (S_ISREG(buf.st_mode))
{
pgFile **prev_file = NULL;
char *extra_path = NULL;
char *external_path = NULL;
if (file->extra_dir_num)
extra_path = parray_get(arguments->extra_dirs,
file->extra_dir_num - 1);
if (file->external_dir_num)
external_path = parray_get(arguments->external_dirs,
file->external_dir_num - 1);
/* Check that file exist in previous backup */
if (current.backup_mode != BACKUP_MODE_FULL)
@ -2304,13 +2308,13 @@ backup_files(void *arg)
char *relative;
pgFile key;
relative = GetRelativePath(file->path, file->extra_dir_num ?
extra_path : arguments->from_root);
relative = GetRelativePath(file->path, file->external_dir_num ?
external_path : arguments->from_root);
key.path = relative;
key.extra_dir_num = file->extra_dir_num;
key.external_dir_num = file->external_dir_num;
prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist,
&key, pgFileComparePathWithExtra);
&key, pgFileComparePathWithExternal);
if (prev_file)
/* File exists in previous backup */
file->exists_in_prev = true;
@ -2335,7 +2339,7 @@ backup_files(void *arg)
continue;
}
}
else if (!file->extra_dir_num &&
else if (!file->external_dir_num &&
strcmp(file->name, "pg_control") == 0)
copy_pgcontrol_file(arguments->from_root, arguments->to_root,
file);
@ -2344,7 +2348,7 @@ backup_files(void *arg)
const char *src;
const char *dst;
bool skip = false;
char extra_dst[MAXPGPATH];
char external_dst[MAXPGPATH];
/* If non-data file has not changed since last backup... */
if (prev_file && file->exists_in_prev &&
@ -2356,12 +2360,13 @@ backup_files(void *arg)
skip = true; /* ...skip copying file. */
}
/* Set file paths */
if (file->extra_dir_num)
if (file->external_dir_num)
{
makeExtraDirPathByNum(extra_dst, arguments->extra_prefix,
file->extra_dir_num);
src = extra_path;
dst = extra_dst;
makeExternalDirPathByNum(external_dst,
arguments->external_prefix,
file->external_dir_num);
src = external_path;
dst = external_dst;
}
else
{

36
src/catalog.c
View File

@ -414,19 +414,21 @@ pgBackupCreateDir(pgBackup *backup)
temp = palloc(strlen(DATABASE_DIR) + 1);
parray_append(subdirs, temp);
/* Add extra dirs containers */
if (backup->extra_dir_str)
/* Add external dirs containers */
if (backup->external_dir_str)
{
parray *extradirs_list = make_extra_directory_list(backup->extra_dir_str);
for (int i = 0; i < parray_num(extradirs_list); i++)
parray *external_list;
external_list = make_external_directory_list(backup->external_dir_str);
for (int i = 0; i < parray_num(external_list); i++)
{
/* 20 chars is enough to hold the extradir number in string. */
temp = palloc(strlen(EXTRA_DIR) + 20);
/* Numeration of extradirs starts with 1 */
makeExtraDirPathByNum(temp, EXTRA_DIR, i+1);
/* 20 chars is enough to hold the externaldir number in string. */
temp = palloc(strlen(EXTERNAL_DIR) + 20);
/* Numeration of externaldirs starts with 1 */
makeExternalDirPathByNum(temp, EXTERNAL_DIR, i+1);
parray_append(subdirs, temp);
}
free_dir_list(extradirs_list);
free_dir_list(external_list);
}
pgBackupGetPath(backup, path, lengthof(path), NULL);
@ -517,9 +519,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
if (backup->primary_conninfo)
fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo);
/* print extra directories list */
if (backup->extra_dir_str)
fprintf(out, "extra-directory = '%s'\n", backup->extra_dir_str);
/* print external directories list */
if (backup->external_dir_str)
fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str);
}
/*
@ -551,7 +553,7 @@ write_backup(pgBackup *backup)
*/
void
write_backup_filelist(pgBackup *backup, parray *files, const char *root,
const char *extra_prefix, parray *extra_list)
const char *external_prefix, parray *external_list)
{
FILE *fp;
char path[MAXPGPATH];
@ -563,7 +565,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
elog(ERROR, "Cannot open file list \"%s\": %s", path,
strerror(errno));
print_file_list(fp, files, root, extra_prefix, extra_list);
print_file_list(fp, files, root, external_prefix, external_list);
if (fflush(fp) != 0 ||
fsync(fileno(fp)) != 0 ||
@ -614,7 +616,7 @@ readBackupControlFile(const char *path)
{'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT},
{'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT},
{'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT},
{'s', 0, "extra-directory", &backup->extra_dir_str, SOURCE_FILE_STRICT},
{'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT},
{0}
};
@ -844,7 +846,7 @@ pgBackupInit(pgBackup *backup)
backup->primary_conninfo = NULL;
backup->program_version[0] = '\0';
backup->server_version[0] = '\0';
backup->extra_dir_str = NULL;
backup->external_dir_str = NULL;
}
/* free pgBackup object */
@ -854,7 +856,7 @@ pgBackupFree(void *backup)
pgBackup *b = (pgBackup *) backup;
pfree(b->primary_conninfo);
pfree(b->extra_dir_str);
pfree(b->external_dir_str);
pfree(backup);
}

4
src/configure.c
View File

@ -62,8 +62,8 @@ ConfigOption instance_options[] =
},
#endif
{
's', 'E', "extra-directory",
&instance_config.extra_dir_str, SOURCE_CMD, 0,
's', 'E', "external-dirs",
&instance_config.external_dir_str, SOURCE_CMD, 0,
OPTION_INSTANCE_GROUP, 0, option_get_value
},
/* Connection options */

106
src/dir.c
View File

@ -122,7 +122,7 @@ static char dir_check_file(const char *root, pgFile *file);
static void dir_list_file_internal(parray *files, const char *root,
pgFile *parent, bool exclude,
bool omit_symlink, parray *black_list,
int extra_dir_num);
int external_dir_num);
static void list_data_directories(parray *files, const char *path, bool is_root,
bool exclude);
@ -133,7 +133,7 @@ static void opt_path_map(ConfigOption *opt, const char *arg,
static TablespaceList tablespace_dirs = {NULL, NULL};
static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL};
/* Extra directories mapping */
static TablespaceList extra_remap_list = {NULL, NULL};
static TablespaceList external_remap_list = {NULL, NULL};
/*
* Create directory, also create parent directories if necessary.
@ -162,7 +162,7 @@ dir_create_dir(const char *dir, mode_t mode)
}
pgFile *
pgFileNew(const char *path, bool omit_symlink, int extra_dir_num)
pgFileNew(const char *path, bool omit_symlink, int external_dir_num)
{
struct stat st;
pgFile *file;
@ -180,7 +180,7 @@ pgFileNew(const char *path, bool omit_symlink, int extra_dir_num)
file = pgFileInit(path);
file->size = st.st_size;
file->mode = st.st_mode;
file->extra_dir_num = extra_dir_num;
file->external_dir_num = external_dir_num;
return file;
}
@ -231,7 +231,7 @@ pgFileInit(const char *path)
/* Number of blocks readed during backup */
file->n_blocks = BLOCKNUM_INVALID;
file->compress_alg = NOT_DEFINED_COMPRESS;
file->extra_dir_num = 0;
file->external_dir_num = 0;
return file;
}
@ -353,11 +353,11 @@ pgFileComparePath(const void *f1, const void *f2)
}
/*
* Compare two pgFile with their path and extra_dir_num
* Compare two pgFile with their path and external_dir_num
* in ascending order of ASCII code.
*/
int
pgFileComparePathWithExtra(const void *f1, const void *f2)
pgFileComparePathWithExternal(const void *f1, const void *f2)
{
pgFile *f1p = *(pgFile **)f1;
pgFile *f2p = *(pgFile **)f2;
@ -366,9 +366,9 @@ pgFileComparePathWithExtra(const void *f1, const void *f2)
res = strcmp(f1p->path, f2p->path);
if (!res)
{
if (f1p->extra_dir_num > f2p->extra_dir_num)
if (f1p->external_dir_num > f2p->external_dir_num)
return 1;
else if (f1p->extra_dir_num < f2p->extra_dir_num)
else if (f1p->external_dir_num < f2p->external_dir_num)
return -1;
else
return 0;
@ -423,7 +423,7 @@ BlackListCompare(const void *str1, const void *str2)
*/
void
dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
bool add_root, int extra_dir_num)
bool add_root, int external_dir_num)
{
pgFile *file;
parray *black_list = NULL;
@ -462,7 +462,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
parray_qsort(black_list, BlackListCompare);
}
file = pgFileNew(root, false, extra_dir_num);
file = pgFileNew(root, false, external_dir_num);
if (file == NULL)
return;
@ -475,7 +475,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
parray_append(files, file);
dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list,
extra_dir_num);
external_dir_num);
if (!add_root)
pgFileFree(file);
@ -686,7 +686,8 @@ dir_check_file(const char *root, pgFile *file)
*/
static void
dir_list_file_internal(parray *files, const char *root, pgFile *parent,
bool exclude, bool omit_symlink, parray *black_list, int extra_dir_num)
bool exclude, bool omit_symlink, parray *black_list,
int external_dir_num)
{
DIR *dir;
struct dirent *dent;
@ -716,7 +717,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
join_path_components(child, parent->path, dent->d_name);
file = pgFileNew(child, omit_symlink, extra_dir_num);
file = pgFileNew(child, omit_symlink, external_dir_num);
if (file == NULL)
continue;
@ -773,7 +774,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
*/
if (S_ISDIR(file->mode))
dir_list_file_internal(files, root, file, exclude, omit_symlink,
black_list, extra_dir_num);
black_list, external_dir_num);
}
if (errno && errno != ENOENT)
@ -986,11 +987,11 @@ opt_tablespace_map(ConfigOption *opt, const char *arg)
opt_path_map(opt, arg, &tablespace_dirs, "tablespace");
}
/* Parse extra directories mapping */
/* Parse external directories mapping */
void
opt_extradir_map(ConfigOption *opt, const char *arg)
opt_externaldir_map(ConfigOption *opt, const char *arg)
{
opt_path_map(opt, arg, &extra_remap_list, "extra directory");
opt_path_map(opt, arg, &external_remap_list, "external directory");
}
/*
@ -1264,31 +1265,31 @@ check_tablespace_mapping(pgBackup *backup)
}
void
check_extra_dir_mapping(pgBackup *backup)
check_external_dir_mapping(pgBackup *backup)
{
TablespaceListCell *cell;
parray *extra_dirs_to_restore;
parray *external_dirs_to_restore;
bool found;
int i;
if (!backup->extra_dir_str)
if (!backup->external_dir_str)
{
if (extra_remap_list.head)
elog(ERROR, "--extra-mapping option's old directory doesn't have "
"an entry in list of extra directories of current "
"backup: \"%s\"", extra_remap_list.head->old_dir);
if (external_remap_list.head)
elog(ERROR, "--external-mapping option's old directory doesn't "
"have an entry in list of external directories of current "
"backup: \"%s\"", external_remap_list.head->old_dir);
return;
}
extra_dirs_to_restore = make_extra_directory_list(backup->extra_dir_str);
for (cell = extra_remap_list.head; cell; cell = cell->next)
external_dirs_to_restore = make_external_directory_list(backup->external_dir_str);
for (cell = external_remap_list.head; cell; cell = cell->next)
{
char *old_dir = cell->old_dir;
found = false;
for (i = 0; i < parray_num(extra_dirs_to_restore); i++)
for (i = 0; i < parray_num(external_dirs_to_restore); i++)
{
char *external_dir = parray_get(extra_dirs_to_restore, i);
char *external_dir = parray_get(external_dirs_to_restore, i);
if (strcmp(old_dir, external_dir) == 0)
{
found = true;
@ -1296,18 +1297,18 @@ check_extra_dir_mapping(pgBackup *backup)
}
}
if (!found)
elog(ERROR, "--extra-mapping option's old directory doesn't have "
"an entry in list of extra directories of current "
elog(ERROR, "--external-mapping option's old directory doesn't "
"have an entry in list of external directories of current "
"backup: \"%s\"", cell->old_dir);
}
}
char *
get_extra_remap(char *current_dir)
get_external_remap(char *current_dir)
{
TablespaceListCell *cell;
for (cell = extra_remap_list.head; cell; cell = cell->next)
for (cell = external_remap_list.head; cell; cell = cell->next)
{
char *old_dir = cell->old_dir;
@ -1322,7 +1323,7 @@ get_extra_remap(char *current_dir)
*/
void
print_file_list(FILE *out, const parray *files, const char *root,
const char *extra_prefix, parray *extra_list)
const char *external_prefix, parray *external_list)
{
size_t i;
@ -1335,20 +1336,20 @@ print_file_list(FILE *out, const parray *files, const char *root,
/* omit root directory portion */
if (root && strstr(path, root) == path)
path = GetRelativePath(path, root);
else if (file->extra_dir_num && !extra_prefix)
else if (file->external_dir_num && !external_prefix)
{
Assert(extra_list);
path = GetRelativePath(path, parray_get(extra_list,
file->extra_dir_num - 1));
Assert(external_list);
path = GetRelativePath(path, parray_get(external_list,
file->external_dir_num - 1));
}
fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
"\"mode\":\"%u\", \"is_datafile\":\"%u\", "
"\"is_cfs\":\"%u\", \"crc\":\"%u\", "
"\"compress_alg\":\"%s\", \"extra_dir_num\":\"%d\"",
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\"",
path, file->write_size, file->mode,
file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc,
deparse_compress_alg(file->compress_alg), file->extra_dir_num);
deparse_compress_alg(file->compress_alg), file->external_dir_num);
if (file->is_datafile)
fprintf(out, ",\"segno\":\"%d\"", file->segno);
@ -1511,7 +1512,8 @@ bad_format:
* If root is not NULL, path will be absolute path.
*/
parray *
dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt)
dir_read_file_list(const char *root, const char *external_prefix,
const char *file_txt)
{
FILE *fp;
parray *files;
@ -1533,7 +1535,7 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_
mode, /* bit length of mode_t depends on platforms */
is_datafile,
is_cfs,
extra_dir_num,
external_dir_num,
crc,
segno,
n_blocks;
@ -1546,13 +1548,13 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_
get_control_value(buf, "is_cfs", NULL, &is_cfs, false);
get_control_value(buf, "crc", NULL, &crc, true);
get_control_value(buf, "compress_alg", compress_alg_string, NULL, false);
get_control_value(buf, "extra_dir_num", NULL, &extra_dir_num, false);
get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false);
if (extra_dir_num && extra_prefix)
if (external_dir_num && external_prefix)
{
char temp[MAXPGPATH];
makeExtraDirPathByNum(temp, extra_prefix, extra_dir_num);
makeExternalDirPathByNum(temp, external_prefix, external_dir_num);
join_path_components(filepath, temp, path);
}
else if (root)
@ -1568,7 +1570,7 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_
file->is_cfs = is_cfs ? true : false;
file->crc = (pg_crc32) crc;
file->compress_alg = parse_compress_alg(compress_alg_string);
file->extra_dir_num = extra_dir_num;
file->external_dir_num = external_dir_num;
/*
* Optional fields
@ -1656,11 +1658,11 @@ pgFileSize(const char *path)
}
/*
* Construct parray containing extra directories paths
* Construct parray containing external directories paths
* from string like /path1:/path2
*/
parray *
make_extra_directory_list(const char *colon_separated_dirs)
make_external_directory_list(const char *colon_separated_dirs)
{
char *p;
parray *list = parray_new();
@ -1693,19 +1695,19 @@ free_dir_list(parray *list)
/* Append to string "path_prefix" int "dir_num" */
void
makeExtraDirPathByNum(char *ret_path, const char *path_prefix,
const int dir_num)
makeExternalDirPathByNum(char *ret_path, const char *path_prefix,
const int dir_num)
{
sprintf(ret_path, "%s%d", path_prefix, dir_num);
}
/* Check if "dir" presents in "dirs_list" */
bool
backup_contains_extra(const char *dir, parray *dirs_list)
backup_contains_external(const char *dir, parray *dirs_list)
{
void *search_result;
if (!dirs_list) /* There is no extra dirs in backup */
if (!dirs_list) /* There is no external dirs in backup */
return false;
search_result = parray_bsearch(dirs_list, dir, BlackListCompare);
return search_result != NULL;

22
src/help.c
View File

@ -119,13 +119,13 @@ help_pg_probackup(void)
printf(_(" [--master-port=port] [--master-user=user_name]\n"));
printf(_(" [--replica-timeout=timeout]\n"));
printf(_(" [--skip-block-validation]\n"));
printf(_(" [--extra-directory=extra-directory-path]\n"));
printf(_(" [--external-dirs=external-directory-path]\n"));
printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n"));
printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n"));
printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n"));
printf(_(" [--extra-mapping=OLDDIR=NEWDIR]\n"));
printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n"));
printf(_(" [--immediate] [--recovery-target-name=target-name]\n"));
printf(_(" [--recovery-target-action=pause|promote|shutdown]\n"));
printf(_(" [--restore-as-replica]\n"));
@ -210,7 +210,7 @@ help_backup(void)
printf(_(" [--master-port=port] [--master-user=user_name]\n"));
printf(_(" [--replica-timeout=timeout]\n"));
printf(_(" [--skip-block-validation]\n"));
printf(_(" [-E extra-directory=extra-directory-path]\n\n"));
printf(_(" [-E external-dirs=external-directory-path]\n\n"));
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n"));
@ -223,7 +223,7 @@ help_backup(void)
printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n"));
printf(_(" --progress show progress\n"));
printf(_(" --skip-block-validation set to validate only file-level checksum\n"));
printf(_(" -E --extra-directory=extra-directory-path\n"));
printf(_(" -E --external-dirs=external-directory-path\n"));
printf(_(" backup some directory not from pgdata \n"));
printf(_("\n Logging options:\n"));
@ -286,7 +286,7 @@ help_restore(void)
printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n"));
printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n"));
printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n"));
printf(_(" [--extra-mapping=OLDDIR=NEWDIR]\n"));
printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n"));
printf(_(" [--immediate] [--recovery-target-name=target-name]\n"));
printf(_(" [--recovery-target-action=pause|promote|shutdown]\n"));
printf(_(" [--restore-as-replica] [--no-validate]\n"));
@ -306,8 +306,8 @@ help_restore(void)
printf(_(" --timeline=timeline recovering into a particular timeline\n"));
printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"));
printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n"));
printf(_(" --extra-mapping=OLDDIR=NEWDIR\n"));
printf(_(" relocate the extra directory from OLDDIR to NEWDIR\n"));
printf(_(" --external-mapping=OLDDIR=NEWDIR\n"));
printf(_(" relocate the external directory from OLDDIR to NEWDIR\n"));
printf(_(" --immediate end recovery as soon as a consistent state is reached\n"));
printf(_(" --recovery-target-name=target-name\n"));
@ -498,11 +498,11 @@ help_set_config(void)
printf(_(" [--master-port=port] [--master-user=user_name]\n"));
printf(_(" [--replica-timeout=timeout]\n"));
printf(_(" [--archive-timeout=timeout]\n"));
printf(_(" [-E extra-directory=extra-directory-path]\n\n"));
printf(_(" [-E external-dirs=external-directory-path]\n\n"));
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
printf(_(" --instance=instance_name name of the instance\n"));
printf(_(" -E --extra-directory=extra-directory-path\n"));
printf(_(" -E --external-dirs=external-directory-path\n"));
printf(_(" backup some directory not from pgdata \n"));
printf(_("\n Logging options:\n"));
@ -570,12 +570,12 @@ help_add_instance(void)
{
printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME);
printf(_(" --instance=instance_name\n"));
printf(_(" -E extra-directory=extra-directory-path\n\n"));
printf(_(" -E external-dirs=external-directory-path\n\n"));
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n"));
printf(_(" --instance=instance_name name of the new instance\n"));
printf(_(" -E --extra-directory=extra-directory-path\n"));
printf(_(" -E --external-dirs=external-directory-path\n"));
printf(_(" backup some directory not from pgdata \n"));
}

125
src/merge.c
View File

@ -18,14 +18,14 @@ typedef struct
{
parray *to_files;
parray *files;
parray *from_extra;
parray *from_external;
pgBackup *to_backup;
pgBackup *from_backup;
const char *to_root;
const char *from_root;
const char *to_extra_prefix;
const char *from_extra_prefix;
const char *to_external_prefix;
const char *from_external_prefix;
/*
* Return value from the thread.
@ -37,9 +37,10 @@ typedef struct
static void merge_backups(pgBackup *backup, pgBackup *next_backup);
static void *merge_files(void *arg);
static void
reorder_extra_dirs(pgBackup *to_backup, parray *to_extra, parray *from_extra);
reorder_external_dirs(pgBackup *to_backup, parray *to_external,
parray *from_external);
static int
get_extra_index(const char *key, const parray *list);
get_external_index(const char *key, const parray *list);
/*
* Implementation of MERGE command.
@ -165,15 +166,15 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
*from_backup_id = base36enc_dup(from_backup->start_time);
char to_backup_path[MAXPGPATH],
to_database_path[MAXPGPATH],
to_extra_prefix[MAXPGPATH],
to_external_prefix[MAXPGPATH],
from_backup_path[MAXPGPATH],
from_database_path[MAXPGPATH],
from_extra_prefix[MAXPGPATH],
from_external_prefix[MAXPGPATH],
control_file[MAXPGPATH];
parray *files,
*to_files;
parray *to_extra = NULL,
*from_extra = NULL;
parray *to_external = NULL,
*from_external = NULL;
pthread_t *threads = NULL;
merge_files_arg *threads_args = NULL;
int i;
@ -209,13 +210,13 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL);
pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path),
DATABASE_DIR);
pgBackupGetPath(to_backup, to_extra_prefix, lengthof(to_database_path),
EXTRA_DIR);
pgBackupGetPath(to_backup, to_external_prefix, lengthof(to_database_path),
EXTERNAL_DIR);
pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL);
pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path),
DATABASE_DIR);
pgBackupGetPath(from_backup, from_extra_prefix, lengthof(from_database_path),
EXTRA_DIR);
pgBackupGetPath(from_backup, from_external_prefix, lengthof(from_database_path),
EXTERNAL_DIR);
/*
* Get list of files which will be modified or removed.
@ -252,32 +253,32 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads);
/* Create extra directories lists */
if (to_backup->extra_dir_str)
to_extra = make_extra_directory_list(to_backup->extra_dir_str);
if (from_backup->extra_dir_str)
from_extra = make_extra_directory_list(from_backup->extra_dir_str);
/* Create external directories lists */
if (to_backup->external_dir_str)
to_external = make_external_directory_list(to_backup->external_dir_str);
if (from_backup->external_dir_str)
from_external = make_external_directory_list(from_backup->external_dir_str);
/*
* Rename extra directoties in to_backup (if exists)
* according to numeration of extra dirs in from_backup.
* Rename external directoties in to_backup (if exists)
* according to numeration of external dirs in from_backup.
*/
if (to_extra)
reorder_extra_dirs(to_backup, to_extra, from_extra);
if (to_external)
reorder_external_dirs(to_backup, to_external, from_external);
/* Setup threads */
for (i = 0; i < parray_num(files); i++)
{
pgFile *file = (pgFile *) parray_get(files, i);
/* if the entry was an extra directory, create it in the backup */
if (file->extra_dir_num && S_ISDIR(file->mode))
/* if the entry was an external directory, create it in the backup */
if (file->external_dir_num && S_ISDIR(file->mode))
{
char dirpath[MAXPGPATH];
char new_container[MAXPGPATH];
makeExtraDirPathByNum(new_container, to_extra_prefix,
file->extra_dir_num);
makeExternalDirPathByNum(new_container, to_external_prefix,
file->external_dir_num);
join_path_components(dirpath, new_container, file->path);
dir_create_dir(dirpath, DIR_PERMISSION);
}
@ -294,9 +295,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
arg->from_backup = from_backup;
arg->to_root = to_database_path;
arg->from_root = from_database_path;
arg->from_extra = from_extra;
arg->to_extra_prefix = to_extra_prefix;
arg->from_extra_prefix = from_extra_prefix;
arg->from_external = from_external;
arg->to_external_prefix = to_external_prefix;
arg->from_external_prefix = from_external_prefix;
/* By default there are some error */
arg->ret = 1;
@ -325,9 +326,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
to_backup->recovery_time = from_backup->recovery_time;
to_backup->recovery_xid = from_backup->recovery_xid;
pfree(to_backup->extra_dir_str);
to_backup->extra_dir_str = from_backup->extra_dir_str;
from_backup->extra_dir_str = NULL; /* For safe pgBackupFree() */
pfree(to_backup->external_dir_str);
to_backup->external_dir_str = from_backup->external_dir_str;
from_backup->external_dir_str = NULL; /* For safe pgBackupFree() */
/*
* If one of the backups isn't "stream" backup then the target backup become
* non-stream backup too.
@ -354,7 +355,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
to_backup->wal_bytes = BYTES_INVALID;
write_backup_filelist(to_backup, files, from_database_path,
from_extra_prefix, NULL);
from_external_prefix, NULL);
write_backup(to_backup);
delete_source_backup:
@ -371,10 +372,10 @@ delete_source_backup:
for (i = 0; i < parray_num(to_files); i++)
{
pgFile *file = (pgFile *) parray_get(to_files, i);
char *dir_name = parray_get(to_external, file->external_dir_num - 1);
if (file->extra_dir_num &&
backup_contains_extra(parray_get(to_extra, file->extra_dir_num - 1),
from_extra))
if (file->external_dir_num &&
backup_contains_external(dir_name, from_external))
/* Dir already removed*/
continue;
@ -490,11 +491,11 @@ merge_files(void *arg)
}
/* We need to make full path, file object has relative path */
if (file->extra_dir_num)
if (file->external_dir_num)
{
char temp[MAXPGPATH];
makeExtraDirPathByNum(temp, argument->from_extra_prefix,
file->extra_dir_num);
makeExternalDirPathByNum(temp, argument->from_external_prefix,
file->external_dir_num);
join_path_components(from_file_path, temp, file->path);
}
@ -612,21 +613,21 @@ merge_files(void *arg)
}
else if (strcmp(file->name, "pg_control") == 0)
copy_pgcontrol_file(argument->from_root, argument->to_root, file);
else if (file->extra_dir_num)
else if (file->external_dir_num)
{
char from_root[MAXPGPATH];
char to_root[MAXPGPATH];
int new_dir_num;
char *file_extra_path = parray_get(argument->from_extra,
file->extra_dir_num - 1);
char *file_external_path = parray_get(argument->from_external,
file->external_dir_num - 1);
Assert(argument->from_extra);
new_dir_num = get_extra_index(file_extra_path,
argument->from_extra);
makeExtraDirPathByNum(from_root, argument->from_extra_prefix,
file->extra_dir_num);
makeExtraDirPathByNum(to_root, argument->to_extra_prefix,
new_dir_num);
Assert(argument->from_external);
new_dir_num = get_external_index(file_external_path,
argument->from_external);
makeExternalDirPathByNum(from_root, argument->from_external_prefix,
file->external_dir_num);
makeExternalDirPathByNum(to_root, argument->to_external_prefix,
new_dir_num);
copy_file(from_root, to_root, file);
}
else
@ -668,9 +669,9 @@ remove_dir_with_files(const char *path)
}
}
/* Get index of extra directory */
/* Get index of external directory */
static int
get_extra_index(const char *key, const parray *list)
get_external_index(const char *key, const parray *list)
{
if (!list) /* Nowhere to search */
return -1;
@ -682,29 +683,31 @@ get_extra_index(const char *key, const parray *list)
return -1;
}
/* Rename directories in to_backup according to order in from_extra */
/* Rename directories in to_backup according to order in from_external */
static void
reorder_extra_dirs(pgBackup *to_backup, parray *to_extra, parray *from_extra)
reorder_external_dirs(pgBackup *to_backup, parray *to_external,
parray *from_external)
{
char extradir_template[MAXPGPATH];
char externaldir_template[MAXPGPATH];
pgBackupGetPath(to_backup, extradir_template,
lengthof(extradir_template), EXTRA_DIR);
for (int i = 0; i < parray_num(to_extra); i++)
pgBackupGetPath(to_backup, externaldir_template,
lengthof(externaldir_template), EXTERNAL_DIR);
for (int i = 0; i < parray_num(to_external); i++)
{
int from_num = get_extra_index(parray_get(to_extra, i), from_extra);
int from_num = get_external_index(parray_get(to_external, i),
from_external);
if (from_num == -1)
{
char old_path[MAXPGPATH];
makeExtraDirPathByNum(old_path, extradir_template, i + 1);
makeExternalDirPathByNum(old_path, externaldir_template, i + 1);
remove_dir_with_files(old_path);
}
else if (from_num != i + 1)
{
char old_path[MAXPGPATH];
char new_path[MAXPGPATH];
makeExtraDirPathByNum(old_path, extradir_template, i + 1);
makeExtraDirPathByNum(new_path, extradir_template, from_num);
makeExternalDirPathByNum(old_path, externaldir_template, i + 1);
makeExternalDirPathByNum(new_path, externaldir_template, from_num);
elog(VERBOSE, "Rename %s to %s", old_path, new_path);
if (rename (old_path, new_path) == -1)
elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s",

6
src/pg_probackup.c
View File

@ -54,8 +54,8 @@ char backup_instance_path[MAXPGPATH];
*/
char arclog_path[MAXPGPATH] = "";
/* colon separated extra directories list ("/path1:/path2") */
char *extradir = NULL;
/* colon separated external directories list ("/path1:/path2") */
char *externaldir = NULL;
/* common options */
static char *backup_id_string = NULL;
int num_threads = 1;
@ -145,7 +145,7 @@ static ConfigOption cmd_options[] =
{ 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT },
{ 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT },
{ 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT },
{ 'f', 155, "extra-mapping", opt_extradir_map, SOURCE_CMD_STRICT },
{ 'f', 155, "external-mapping", opt_externaldir_map, SOURCE_CMD_STRICT },
{ 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT },
{ 's', 141, "recovery-target-name", &target_name, SOURCE_CMD_STRICT },
{ 's', 142, "recovery-target-action", &target_action, SOURCE_CMD_STRICT },

42
src/pg_probackup.h
View File

@ -48,7 +48,7 @@
#define PG_BACKUP_LABEL_FILE "backup_label"
#define PG_BLACK_LIST "black_list"
#define PG_TABLESPACE_MAP_FILE "tablespace_map"
#define EXTRA_DIR "extra_directories/extradir"
#define EXTERNAL_DIR "external_directories/externaldir"
/* Timeout defaults */
#define ARCHIVE_TIMEOUT_DEFAULT 300
@ -122,7 +122,7 @@ typedef struct pgFile
int n_blocks; /* size of the file in blocks, readed during DELTA backup */
bool is_cfs; /* Flag to distinguish files compressed by CFS*/
bool is_database;
int extra_dir_num; /* Number of extra directory. 0 if not extra */
int external_dir_num; /* Number of external directory. 0 if not external */
bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */
CompressAlg compress_alg; /* compression algorithm applied to the file */
volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */
@ -180,7 +180,7 @@ typedef struct InstanceConfig
uint32 xlog_seg_size;
char *pgdata;
char *extra_dir_str;
char *external_dir_str;
const char *pgdatabase;
const char *pghost;
const char *pgport;
@ -261,7 +261,8 @@ struct pgBackup
pgBackup *parent_backup_link;
char *primary_conninfo; /* Connection parameters of the backup
* in the format suitable for recovery.conf */
char *extra_dir_str; /* List of extra directories, separated by ':' */
char *external_dir_str; /* List of external directories,
* separated by ':' */
};
/* Recovery target for restore and validate subcommands */
@ -292,11 +293,11 @@ typedef struct
{
const char *from_root;
const char *to_root;
const char *extra_prefix;
const char *external_prefix;
parray *files_list;
parray *prev_filelist;
parray *extra_dirs;
parray *external_dirs;
XLogRecPtr prev_start_lsn;
PGconn *backup_conn;
@ -474,8 +475,8 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
extern void catalog_lock(void);
extern void pgBackupWriteControl(FILE *out, pgBackup *backup);
extern void write_backup_filelist(pgBackup *backup, parray *files,
const char *root, const char *extra_prefix,
parray *extra_list);
const char *root, const char *external_prefix,
parray *external_list);
extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len,
const char *subdir);
@ -500,26 +501,27 @@ extern const char* deparse_compress_alg(int alg);
/* in dir.c */
extern void dir_list_file(parray *files, const char *root, bool exclude,
bool omit_symlink, bool add_root, int extra_dir_num);
bool omit_symlink, bool add_root, int external_dir_num);
extern void create_data_directories(const char *data_dir,
const char *backup_dir,
bool extract_tablespaces);
extern void read_tablespace_map(parray *files, const char *backup_dir);
extern void opt_tablespace_map(ConfigOption *opt, const char *arg);
extern void opt_extradir_map(ConfigOption *opt, const char *arg);
extern void opt_externaldir_map(ConfigOption *opt, const char *arg);
extern void check_tablespace_mapping(pgBackup *backup);
extern void check_extra_dir_mapping(pgBackup *backup);
extern char *get_extra_remap(char *current_dir);
extern void check_external_dir_mapping(pgBackup *backup);
extern char *get_external_remap(char *current_dir);
extern void print_file_list(FILE *out, const parray *files, const char *root,
const char *extra_prefix, parray *extra_list);
extern parray *dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt);
extern parray *make_extra_directory_list(const char *colon_separated_dirs);
const char *external_prefix, parray *external_list);
extern parray *dir_read_file_list(const char *root, const char *external_prefix,
const char *file_txt);
extern parray *make_external_directory_list(const char *colon_separated_dirs);
extern void free_dir_list(parray *list);
extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path,
const int dir_num);
extern bool backup_contains_extra(const char *dir, parray *dirs_list);
extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path,
const int dir_num);
extern bool backup_contains_external(const char *dir, parray *dirs_list);
extern int dir_create_dir(const char *path, mode_t mode);
extern bool dir_is_empty(const char *path);
@ -527,14 +529,14 @@ extern bool dir_is_empty(const char *path);
extern bool fileExists(const char *path);
extern size_t pgFileSize(const char *path);
extern pgFile *pgFileNew(const char *path, bool omit_symlink, int extra_dir_num);
extern pgFile *pgFileNew(const char *path, bool omit_symlink, int external_dir_num);
extern pgFile *pgFileInit(const char *path);
extern void pgFileDelete(pgFile *file);
extern void pgFileFree(void *file);
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c,
bool raise_on_deleted, size_t *bytes_read);
extern int pgFileComparePath(const void *f1, const void *f2);
extern int pgFileComparePathWithExtra(const void *f1, const void *f2);
extern int pgFileComparePathWithExternal(const void *f1, const void *f2);
extern int pgFileComparePathDesc(const void *f1, const void *f2);
extern int pgFileCompareLinked(const void *f1, const void *f2);
extern int pgFileCompareSize(const void *f1, const void *f2);

93
src/restore.c
View File

@ -21,9 +21,9 @@ typedef struct
{
parray *files;
pgBackup *backup;
parray *req_extra_dirs;
parray *cur_extra_dirs;
char *extra_prefix;
parray *req_external_dirs;
parray *cur_external_dirs;
char *external_prefix;
/*
* Return value from the thread.
@ -32,7 +32,7 @@ typedef struct
int ret;
} restore_files_arg;
static void restore_backup(pgBackup *backup, const char *extra_dir_str);
static void restore_backup(pgBackup *backup, const char *external_dir_str);
static void create_recovery_conf(time_t backup_id,
pgRecoveryTarget *rt,
pgBackup *backup);
@ -287,7 +287,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
if (is_restore)
{
check_tablespace_mapping(dest_backup);
check_extra_dir_mapping(dest_backup);
check_external_dir_mapping(dest_backup);
}
if (!is_restore || !rt->restore_no_validate)
@ -393,7 +393,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn",
base36enc(dest_backup->start_time), dest_backup->server_version);
restore_backup(backup, dest_backup->extra_dir_str);
restore_backup(backup, dest_backup->external_dir_str);
}
/*
@ -420,16 +420,16 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
* Restore one backup.
*/
void
restore_backup(pgBackup *backup, const char *extra_dir_str)
restore_backup(pgBackup *backup, const char *external_dir_str)
{
char timestamp[100];
char this_backup_path[MAXPGPATH];
char database_path[MAXPGPATH];
char extra_prefix[MAXPGPATH];
char external_prefix[MAXPGPATH];
char list_path[MAXPGPATH];
parray *files;
parray *requested_extra_dirs = NULL;
parray *current_extra_dirs = NULL;
parray *requested_external_dirs = NULL;
parray *current_external_dirs = NULL;
int i;
/* arrays with meta info for multi threaded backup */
pthread_t *threads;
@ -461,61 +461,63 @@ restore_backup(pgBackup *backup, const char *extra_dir_str)
pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL);
create_data_directories(instance_config.pgdata, this_backup_path, true);
if(extra_dir_str)
if(external_dir_str)
{
requested_extra_dirs = make_extra_directory_list(extra_dir_str);
for (i = 0; i < parray_num(requested_extra_dirs); i++)
requested_external_dirs = make_external_directory_list(external_dir_str);
for (i = 0; i < parray_num(requested_external_dirs); i++)
{
char *extra_path = parray_get(requested_extra_dirs, i);
extra_path = get_extra_remap(extra_path);
dir_create_dir(extra_path, DIR_PERMISSION);
char *external_path = parray_get(requested_external_dirs, i);
external_path = get_external_remap(external_path);
dir_create_dir(external_path, DIR_PERMISSION);
}
}
if(backup->extra_dir_str)
current_extra_dirs = make_extra_directory_list(backup->extra_dir_str);
if(backup->external_dir_str)
current_external_dirs = make_external_directory_list(backup->external_dir_str);
/*
* Get list of files which need to be restored.
*/
pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR);
pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR);
pgBackupGetPath(backup, external_prefix, lengthof(external_prefix),
EXTERNAL_DIR);
pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST);
files = dir_read_file_list(database_path, extra_prefix, list_path);
files = dir_read_file_list(database_path, external_prefix, list_path);
/* Restore directories in do_backup_instance way */
parray_qsort(files, pgFileComparePath);
/*
* Make extra directories before restore
* Make external directories before restore
* and setup threads at the same time
*/
for (i = 0; i < parray_num(files); i++)
{
pgFile *file = (pgFile *) parray_get(files, i);
/* if the entry was an extra directory, create it in the backup */
if (file->extra_dir_num && S_ISDIR(file->mode))
/* if the entry was an external directory, create it in the backup */
if (file->external_dir_num && S_ISDIR(file->mode))
{
char dirpath[MAXPGPATH];
char *dir_name;
char *extra_path;
char *external_path;
if (!current_extra_dirs ||
parray_num(current_extra_dirs) < file->extra_dir_num - 1)
elog(ERROR, "Inconsistent extra directory backup metadata");
if (!current_external_dirs ||
parray_num(current_external_dirs) < file->external_dir_num - 1)
elog(ERROR, "Inconsistent external directory backup metadata");
extra_path = parray_get(current_extra_dirs, file->extra_dir_num - 1);
if (backup_contains_extra(extra_path, requested_extra_dirs))
external_path = parray_get(current_external_dirs,
file->external_dir_num - 1);
if (backup_contains_external(external_path, requested_external_dirs))
{
char container_dir[MAXPGPATH];
extra_path = get_extra_remap(extra_path);
makeExtraDirPathByNum(container_dir, extra_prefix,
file->extra_dir_num);
external_path = get_external_remap(external_path);
makeExternalDirPathByNum(container_dir, external_prefix,
file->external_dir_num);
dir_name = GetRelativePath(file->path, container_dir);
elog(VERBOSE, "Create directory \"%s\"", dir_name);
join_path_components(dirpath, extra_path, dir_name);
join_path_components(dirpath, external_path, dir_name);
dir_create_dir(dirpath, DIR_PERMISSION);
}
}
@ -533,9 +535,9 @@ restore_backup(pgBackup *backup, const char *extra_dir_str)
arg->files = files;
arg->backup = backup;
arg->req_extra_dirs = requested_extra_dirs;
arg->cur_extra_dirs = current_extra_dirs;
arg->extra_prefix = extra_prefix;
arg->req_external_dirs = requested_external_dirs;
arg->cur_external_dirs = current_external_dirs;
arg->external_prefix = external_prefix;
/* By default there are some error */
threads_args[i].ret = 1;
@ -578,13 +580,13 @@ remove_deleted_files(pgBackup *backup)
parray *files;
parray *files_restored;
char filelist_path[MAXPGPATH];
char extra_prefix[MAXPGPATH];
char external_prefix[MAXPGPATH];
int i;
pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST);
pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR);
pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), EXTERNAL_DIR);
/* Read backup's filelist using target database path as base path */
files = dir_read_file_list(instance_config.pgdata, extra_prefix, filelist_path);
files = dir_read_file_list(instance_config.pgdata, external_prefix, filelist_path);
parray_qsort(files, pgFileComparePathDesc);
/* Get list of files actually existing in target database */
@ -693,14 +695,15 @@ restore_files(void *arg)
false,
parse_program_version(arguments->backup->program_version));
}
else if (file->extra_dir_num)
else if (file->external_dir_num)
{
char *extra_path = parray_get(arguments->cur_extra_dirs,
file->extra_dir_num - 1);
if (backup_contains_extra(extra_path, arguments->req_extra_dirs))
char *external_path = parray_get(arguments->cur_external_dirs,
file->external_dir_num - 1);
if (backup_contains_external(external_path,
arguments->req_external_dirs))
{
extra_path = get_extra_remap(extra_path);
copy_file(arguments->extra_prefix, extra_path, file);
external_path = get_external_remap(external_path);
copy_file(arguments->external_prefix, external_path, file);
}
}
else if (strcmp(file->name, "pg_control") == 0)

8
src/validate.c
View File

@ -43,7 +43,7 @@ void
pgBackupValidate(pgBackup *backup)
{
char base_path[MAXPGPATH];
char extra_prefix[MAXPGPATH];
char external_prefix[MAXPGPATH];
char path[MAXPGPATH];
parray *files;
bool corrupted = false;
@ -90,9 +90,9 @@ pgBackupValidate(pgBackup *backup)
elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time));
pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR);
pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR);
pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), EXTERNAL_DIR);
pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST);
files = dir_read_file_list(base_path, extra_prefix, path);
files = dir_read_file_list(base_path, external_prefix, path);
/* setup threads */
for (i = 0; i < parray_num(files); i++)
@ -240,7 +240,7 @@ pgBackupValidateFiles(void *arg)
*/
if (arguments->backup_version >= 20025 &&
strcmp(file->name, "pg_control") == 0 &&
!file->extra_dir_num)
!file->external_dir_num)
crc = get_pgcontrol_checksum(arguments->base_path);
else
crc = pgFileGetCRC(file->path,

34
tests/external.py
View File

@ -29,13 +29,13 @@ class ExternalTest(ProbackupTest, unittest.TestCase):
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
external_dir = os.path.join(node.base_dir, 'somedirectory')
# copy postgresql.conf to extra_directory
# copy postgresql.conf to external_directory
os.mkdir(external_dir)
shutil.copyfile(
os.path.join(node.data_dir, 'postgresql.conf'),
os.path.join(external_dir, 'postgresql.conf'))
# create directory in extra_directory
# create directory in external_directory
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
@ -44,7 +44,7 @@ class ExternalTest(ProbackupTest, unittest.TestCase):
backup_id = self.backup_node(
backup_dir, 'node', node,
options=[
'--extra-directory={0}'.format(external_dir)])
'--external-dirs={0}'.format(external_dir)])
if self.paranoia:
pgdata = self.pgdata_content(
@ -93,7 +93,7 @@ class ExternalTest(ProbackupTest, unittest.TestCase):
external_dir1_old = self.get_tblspace_path(node, 'external_dir1')
external_dir2_old = self.get_tblspace_path(node, 'external_dir2')
# copy postgresql.conf to extra_directory
# copy postgresql.conf to external_directory
os.mkdir(external_dir1_old)
os.mkdir(external_dir2_old)
shutil.copyfile(
@ -115,9 +115,9 @@ class ExternalTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"--extra-mapping={0}={1}".format(
"--external-mapping={0}={1}".format(
external_dir1_old, external_dir1_new),
"--extra-mapping={0}={1}".format(
"--external-mapping={0}={1}".format(
external_dir2_old, external_dir2_new)])
# we should die here because exception is what we expect to happen
self.assertEqual(
@ -127,8 +127,8 @@ class ExternalTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: --extra-mapping option' in e.message and
'have an entry in list of extra directories' in e.message,
'ERROR: --external-mapping option' in e.message and
'have an entry in list of external directories' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
@ -142,9 +142,9 @@ class ExternalTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"--extra-mapping={0}={1}".format(
"--external-mapping={0}={1}".format(
external_dir1_old, external_dir1_new),
"--extra-mapping={0}={1}".format(
"--external-mapping={0}={1}".format(
external_dir2_old, external_dir2_new)])
if self.paranoia:
@ -223,14 +223,14 @@ class ExternalTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# extra directory contain symlink to file
# extra directory contain symlink to directory
# latest page backup without extra_dir
# external directory contain symlink to file
# external directory contain symlink to directory
# latest page backup without external_dir
# multiple external directories
# --extra-directory=none
# --extra-directory point to a file
# extra directory in config and in command line
# extra directory contain multuple directories, some of them my be empty
# --external-dirs=none
# --external-dirs point to a file
# external directory in config and in command line
# external directory contain multuple directories, some of them my be empty
# forbid to external-dirs to point to tablespace directories
# check that not changed files are not copied by next backup
# merge