From bcf09b3e1947aba3c643a46fd357019cacae714d Mon Sep 17 00:00:00 2001
From: Ivan Kartyshov <i.kartyshov@postgrespro.ru>
Date: Tue, 5 Jun 2018 09:47:49 +0300
Subject: [PATCH] Using option --extra-directory=/path/to/directory You can
add files to backup from extra directory (excluding subdirectory) 1) The
permissons of extra directory must allow to read/write in it 2) You can add
only one directory 3) No additional subfolders creates in backup/database/
TODO: 1) to add more then one extra directory 2) make to backup extra
directory subdirectories
---
src/backup.c | 10 ++++++----
src/delete.c | 2 +-
src/dir.c | 41 ++++++++++++++++++++++++++++-------------
src/pg_probackup.c | 3 +++
src/pg_probackup.h | 9 +++++++--
src/restore.c | 4 +++-
6 files changed, 48 insertions(+), 21 deletions(-)
diff --git a/src/backup.c b/src/backup.c
index a0b02fcf..f00bf950 100644
--- a/src/backup.c
+++ b/src/backup.c
@@ -603,11 +603,13 @@ do_backup_instance(void)
if (is_remote_backup)
get_remote_pgdata_filelist(backup_files_list);
else
- dir_list_file(backup_files_list, pgdata, true, true, false);
+ dir_list_file(backup_files_list, pgdata, true, true, false, false);
/* Extract information about files in backup_list parsing their names:*/
parse_backup_filelist_filenames(backup_files_list, pgdata);
+ dir_list_file(backup_files_list, extradir, true, true, false, true);
+
if (current.backup_mode != BACKUP_MODE_FULL)
{
elog(LOG, "current_tli:%X", current.tli);
@@ -753,7 +755,7 @@ do_backup_instance(void)
/* Scan backup PG_XLOG_DIR */
xlog_files_list = parray_new();
join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
- dir_list_file(xlog_files_list, pg_xlog_path, false, true, false);
+ dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false);
for (i = 0; i < parray_num(xlog_files_list); i++)
{
@@ -1819,7 +1821,7 @@ pg_stop_backup(pgBackup *backup)
*/
if (backup_files_list)
{
- file = pgFileNew(backup_label, true);
+ file = pgFileNew(backup_label, true, false);
calc_file_checksum(file);
free(file->path);
file->path = strdup(PG_BACKUP_LABEL_FILE);
@@ -1863,7 +1865,7 @@ pg_stop_backup(pgBackup *backup)
if (backup_files_list)
{
- file = pgFileNew(tablespace_map, true);
+ file = pgFileNew(tablespace_map, true, false);
if (S_ISREG(file->mode))
calc_file_checksum(file);
free(file->path);
diff --git a/src/delete.c b/src/delete.c
index f81fe70d..cd3c699a 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -271,7 +271,7 @@ pgBackupDeleteFiles(pgBackup *backup)
/* list files to be deleted */
files = parray_new();
pgBackupGetPath(backup, path, lengthof(path), NULL);
- dir_list_file(files, path, false, true, true);
+ dir_list_file(files, path, false, true, true, false);
/* delete leaf node first */
parray_qsort(files, pgFileComparePathDesc);
diff --git a/src/dir.c b/src/dir.c
index 8df3da3f..dd7120f8 100644
--- a/src/dir.c
+++ b/src/dir.c
@@ -93,7 +93,7 @@ static int BlackListCompare(const void *str1, const void *str2);
static bool dir_check_file(const char *root, pgFile *file);
static void dir_list_file_internal(parray *files, const char *root,
pgFile *parent, bool exclude,
- bool omit_symlink, parray *black_list);
+ bool omit_symlink, parray *black_list, bool is_extra);
/*
* Create directory, also create parent directories if necessary.
@@ -123,7 +123,7 @@ dir_create_dir(const char *dir, mode_t mode)
}
pgFile *
-pgFileNew(const char *path, bool omit_symlink)
+pgFileNew(const char *path, bool omit_symlink, bool is_extra)
{
struct stat st;
pgFile *file;
@@ -141,6 +141,8 @@ pgFileNew(const char *path, bool omit_symlink)
file = pgFileInit(path);
file->size = st.st_size;
file->mode = st.st_mode;
+ file->is_extra = is_extra;
+ file->extradir = NULL;
return file;
}
@@ -335,7 +337,7 @@ BlackListCompare(const void *str1, const void *str2)
*/
void
dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
- bool add_root)
+ bool add_root, bool is_extra)
{
pgFile *file;
parray *black_list = NULL;
@@ -372,7 +374,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
parray_qsort(black_list, BlackListCompare);
}
- file = pgFileNew(root, false);
+ file = pgFileNew(root, false, is_extra);
if (file == NULL)
return;
@@ -384,7 +386,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
if (add_root)
parray_append(files, file);
- dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list);
+ dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, is_extra);
parray_qsort(files, pgFileComparePath);
}
@@ -573,7 +575,7 @@ dir_check_file(const char *root, pgFile *file)
*/
static void
dir_list_file_internal(parray *files, const char *root, pgFile *parent,
- bool exclude, bool omit_symlink, parray *black_list)
+ bool exclude, bool omit_symlink, parray *black_list, bool is_extra)
{
DIR *dir;
struct dirent *dent;
@@ -602,7 +604,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
join_path_components(child, parent->path, dent->d_name);
- file = pgFileNew(child, omit_symlink);
+ file = pgFileNew(child, omit_symlink, is_extra);
if (file == NULL)
continue;
@@ -648,7 +650,14 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
/* At least add the file */
if (S_ISREG(file->mode))
+ {
+ if (is_extra)
+ {
+ file->extradir = pgut_strdup(file->path);
+ dirname(file->extradir);
+ }
parray_append(files, file);
+ }
/*
* If the entry is a directory call dir_list_file_internal()
@@ -656,7 +665,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
*/
if (S_ISDIR(file->mode))
dir_list_file_internal(files, root, file, exclude, omit_symlink,
- black_list);
+ black_list, is_extra);
}
if (errno && errno != ENOENT)
@@ -736,7 +745,7 @@ list_data_directories(parray *files, const char *path, bool is_root,
{
pgFile *dir;
- dir = pgFileNew(path, false);
+ dir = pgFileNew(path, false, false);
parray_append(files, dir);
}
@@ -819,10 +828,10 @@ print_file_list(FILE *out, const parray *files, const char *root)
fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\","
"\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\","
- "\"compress_alg\":\"%s\"",
+ "\"compress_alg\":\"%s\", \"is_extra\":\"%u\"",
path, (unsigned long) file->write_size, file->mode,
file->is_datafile?1:0, file->is_cfs?1:0, file->crc,
- deparse_compress_alg(file->compress_alg));
+ deparse_compress_alg(file->compress_alg), file->is_extra?1:0);
if (file->is_datafile)
fprintf(out, ",\"segno\":\"%d\"", file->segno);
@@ -999,6 +1008,7 @@ dir_read_file_list(const char *root, const char *file_txt)
mode, /* bit length of mode_t depends on platforms */
is_datafile,
is_cfs,
+ is_extra,
crc,
segno,
n_blocks;
@@ -1016,14 +1026,20 @@ dir_read_file_list(const char *root, const char *file_txt)
get_control_value(buf, "segno", NULL, &segno, false);
get_control_value(buf, "compress_alg", compress_alg_string, NULL, false);
get_control_value(buf, "n_blocks", NULL, &n_blocks, false);
+ get_control_value(buf, "is_extra", NULL, &is_extra, false);
if (root)
- join_path_components(filepath, root, path);
+ if (is_extra)
+ join_path_components(filepath, root, basename(path));
+ else
+ join_path_components(filepath, root, path);
else
strcpy(filepath, path);
file = pgFileInit(filepath);
+ file->is_extra = is_extra ? true : false;
+ file->extradir = is_extra ? pgut_strdup(dirname(path)) : NULL;
file->write_size = (size_t) write_size;
file->mode = (mode_t) mode;
file->is_datafile = is_datafile ? true : false;
@@ -1034,7 +1050,6 @@ dir_read_file_list(const char *root, const char *file_txt)
file->linked = pgut_strdup(linked);
file->segno = (int) segno;
file->n_blocks = (int) n_blocks;
-
parray_append(files, file);
}
diff --git a/src/pg_probackup.c b/src/pg_probackup.c
index 5d464171..2b63d82d 100644
--- a/src/pg_probackup.c
+++ b/src/pg_probackup.c
@@ -35,6 +35,8 @@ char backup_instance_path[MAXPGPATH];
*/
char arclog_path[MAXPGPATH] = "";
+/* extra directory to backup */
+char *extradir = NULL;
/* common options */
char *backup_id_string_param = NULL;
int num_threads = 1;
@@ -174,6 +176,7 @@ static pgut_option options[] =
/* other options */
{ 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT },
{ 's', 151, "instance", &instance_name, SOURCE_CMDLINE },
+ { 's', 152, "extra-directory", &extradir, SOURCE_CMDLINE },
/* archive-push options */
{ 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE },
{ 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE },
diff --git a/src/pg_probackup.h b/src/pg_probackup.h
index 30df34ce..eda18ede 100644
--- a/src/pg_probackup.h
+++ b/src/pg_probackup.h
@@ -100,6 +100,8 @@ typedef struct pgFile
int n_blocks; /* size of the file in blocks, readed during DELTA backup */
bool is_cfs; /* Flag to distinguish files compressed by CFS*/
bool is_database;
+ bool is_extra;
+ char *extradir; /* File from extra directory */
bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */
CompressAlg compress_alg; /* compression algorithm applied to the file */
volatile uint32 lock; /* lock for synchronization of parallel threads */
@@ -298,6 +300,9 @@ extern char backup_instance_path[MAXPGPATH];
extern char *pgdata;
extern char arclog_path[MAXPGPATH];
+/* extra directory to backup */
+extern char *extradir;
+
/* common options */
extern int num_threads;
extern bool stream_wal;
@@ -442,7 +447,7 @@ extern int pgBackupCompareIdDesc(const void *f1, const void *f2);
/* in dir.c */
extern void dir_list_file(parray *files, const char *root, bool exclude,
- bool omit_symlink, bool add_root);
+ bool omit_symlink, bool add_root, bool is_extra);
extern void list_data_directories(parray *files, const char *path,
bool is_root, bool exclude);
@@ -456,7 +461,7 @@ extern bool dir_is_empty(const char *path);
extern bool fileExists(const char *path);
-extern pgFile *pgFileNew(const char *path, bool omit_symlink);
+extern pgFile *pgFileNew(const char *path, bool omit_symlink, bool is_extra);
extern pgFile *pgFileInit(const char *path);
extern void pgFileDelete(pgFile *file);
extern void pgFileFree(void *file);
diff --git a/src/restore.c b/src/restore.c
index 69bac841..40c78896 100644
--- a/src/restore.c
+++ b/src/restore.c
@@ -461,7 +461,7 @@ remove_deleted_files(pgBackup *backup)
/* Get list of files actually existing in target database */
files_restored = parray_new();
- dir_list_file(files_restored, pgdata, true, true, false);
+ dir_list_file(files_restored, pgdata, true, true, false, false);
/* To delete from leaf, sort in reversed order */
parray_qsort(files_restored, pgFileComparePathDesc);
@@ -768,6 +768,8 @@ restore_files(void *arg)
elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0);
if (file->is_datafile && !file->is_cfs)
restore_data_file(from_root, pgdata, file, arguments->backup);
+ else if (file->is_extra)
+ copy_file(from_root, file->extradir, file);
else
copy_file(from_root, pgdata, file);