db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-07 13:40:17 +02:00
Code Issues Releases Activity

[Issue #169] Major refactoring of backup, restore and merge internal algorihtms

Browse Source
This commit is contained in:
Grigory Smolkin 2020-02-18 19:59:57 +03:00
parent e1b0f1450f
commit 9c662d5622
11 changed files with 1562 additions and 1557 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

281
src/backup.c
View File

@ -80,7 +80,7 @@ static void backup_cleanup(bool fatal, void *userdata);
static void *backup_files(void *arg);
static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo);
static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync);
static void pg_start_backup(const char *label, bool smooth, pgBackup *backup,
PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn);
@ -129,7 +129,7 @@ backup_stopbackup_callback(bool fatal, void *userdata)
* Move files from 'pgdata' to a subdirectory in 'backup_path'.
*/
static void
do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
{
int i;
char database_path[MAXPGPATH];
@ -155,6 +155,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
/* for fancy reporting */
time_t start_time, end_time;
char pretty_time[20];
char pretty_bytes[20];
elog(LOG, "Database backup start");
@ -452,7 +453,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
parray_qsort(backup_files_list, pgFileCompareSize);
/* Sort the array for binary search */
if (prev_backup_filelist)
parray_qsort(prev_backup_filelist, pgFileComparePathWithExternal);
parray_qsort(prev_backup_filelist, pgFileCompareRelPathWithExternal);
/* write initial backup_content.control file and update backup.control */
write_backup_filelist(&current, backup_files_list,
@ -485,6 +486,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
/* Run threads */
thread_interrupted = false;
elog(INFO, "Start transferring data files");
time(&start_time);
for (i = 0; i < num_threads; i++)
{
backup_files_arg *arg = &(threads_args[i]);
@ -500,10 +502,16 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
if (threads_args[i].ret == 1)
backup_isok = false;
}
time(&end_time);
pretty_time_interval(difftime(end_time, start_time),
pretty_time, lengthof(pretty_time));
if (backup_isok)
elog(INFO, "Data files are transferred");
elog(INFO, "Data files are transferred, time elapsed: %s",
pretty_time);
else
elog(ERROR, "Data files transferring failed");
elog(ERROR, "Data files transferring failed, time elapsed: %s",
pretty_time);
/* Remove disappeared during backup files from backup_list */
for (i = 0; i < parray_num(backup_files_list); i++)
@ -589,7 +597,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
{
char *ptr = file->path;
file->path = pstrdup(GetRelativePath(ptr, database_path));
file->path = pgut_strdup(GetRelativePath(ptr, database_path));
file->rel_path = pgut_strdup(file->path);
free(ptr);
}
}
@ -613,6 +622,48 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
/* update backup control file to update size info */
write_backup(&current);
/* Sync all copied files unless '--no-sync' flag is used */
if (no_sync)
elog(WARNING, "Backup files are not synced to disk");
else
{
elog(INFO, "Syncing backup files to disk");
time(&start_time);
for (i = 0; i < parray_num(backup_files_list); i++)
{
char to_fullpath[MAXPGPATH];
pgFile *file = (pgFile *) parray_get(backup_files_list, i);
/* TODO: sync directory ? */
if (S_ISDIR(file->mode))
continue;
if (file->write_size <= 0)
continue;
/* construct fullpath */
if (file->external_dir_num == 0)
join_path_components(to_fullpath, database_path, file->rel_path);
else
{
char external_dst[MAXPGPATH];
makeExternalDirPathByNum(external_dst, external_prefix,
file->external_dir_num);
join_path_components(to_fullpath, external_dst, file->rel_path);
}
if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0)
elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno));
}
time(&end_time);
pretty_time_interval(difftime(end_time, start_time),
pretty_time, lengthof(pretty_time));
elog(INFO, "Backup files are synced, time elapsed: %s", pretty_time);
}
/* clean external directories list */
if (external_dirs)
free_dir_list(external_dirs);
@ -696,7 +747,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
*/
int
do_backup(time_t start_time, bool no_validate,
pgSetBackupParams *set_backup_params)
pgSetBackupParams *set_backup_params, bool no_sync)
{
PGconn *backup_conn = NULL;
PGNodeInfo nodeInfo;
@ -795,7 +846,7 @@ do_backup(time_t start_time, bool no_validate,
elog(ERROR, "Options for connection to master must be provided to perform backup from replica");
/* backup data */
do_backup_instance(backup_conn, &nodeInfo);
do_backup_instance(backup_conn, &nodeInfo, no_sync);
pgut_atexit_pop(backup_cleanup, NULL);
/* compute size of wal files of this backup stored in the archive */
@ -1928,18 +1979,24 @@ static void *
backup_files(void *arg)
{
int i;
backup_files_arg *arguments = (backup_files_arg *) arg;
int n_backup_files_list = parray_num(arguments->files_list);
char from_fullpath[MAXPGPATH];
char to_fullpath[MAXPGPATH];
static time_t prev_time;
backup_files_arg *arguments = (backup_files_arg *) arg;
int n_backup_files_list = parray_num(arguments->files_list);
prev_time = current.start_time;
/* backup a file */
for (i = 0; i < n_backup_files_list; i++)
{
int ret;
struct stat buf;
pgFile *file = (pgFile *) parray_get(arguments->files_list, i);
pgFile *file = (pgFile *) parray_get(arguments->files_list, i);
pgFile *prev_file = NULL;
/* We have already copied all directories */
if (S_ISDIR(file->mode))
continue;
if (arguments->thread_num == 1)
{
@ -1957,7 +2014,6 @@ backup_files(void *arg)
if (!pg_atomic_test_set_flag(&file->lock))
continue;
elog(VERBOSE, "Copying file: \"%s\"", file->path);
/* check for interrupt */
if (interrupted || thread_interrupted)
@ -1965,139 +2021,84 @@ backup_files(void *arg)
if (progress)
elog(INFO, "Progress: (%d/%d). Process file \"%s\"",
i + 1, n_backup_files_list, file->path);
i + 1, n_backup_files_list, file->rel_path);
/* stat file to check its current state */
ret = fio_stat(file->path, &buf, true, FIO_DB_HOST);
if (ret == -1)
/* Handle zero sized files */
if (file->size == 0)
{
if (errno == ENOENT)
{
/*
* If file is not found, this is not en error.
* It could have been deleted by concurrent postgres transaction.
*/
file->write_size = FILE_NOT_FOUND;
elog(LOG, "File \"%s\" is not found", file->path);
continue;
}
else
{
elog(ERROR,
"can't stat file to backup \"%s\": %s",
file->path, strerror(errno));
}
file->write_size = 0;
continue;
}
/* We have already copied all directories */
if (S_ISDIR(buf.st_mode))
continue;
if (S_ISREG(buf.st_mode))
/* construct destination filepath */
if (file->external_dir_num == 0)
{
pgFile **prev_file = NULL;
char *external_path = NULL;
if (file->external_dir_num)
external_path = parray_get(arguments->external_dirs,
file->external_dir_num - 1);
/* Check that file exist in previous backup */
if (current.backup_mode != BACKUP_MODE_FULL)
{
char *relative;
pgFile key;
relative = GetRelativePath(file->path, file->external_dir_num ?
external_path : arguments->from_root);
key.path = relative;
key.external_dir_num = file->external_dir_num;
prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist,
&key, pgFileComparePathWithExternal);
if (prev_file)
/* File exists in previous backup */
file->exists_in_prev = true;
}
/* copy the file into backup */
if (file->is_datafile && !file->is_cfs)
{
char to_path[MAXPGPATH];
join_path_components(to_path, arguments->to_root,
file->path + strlen(arguments->from_root) + 1);
if (current.backup_mode != BACKUP_MODE_FULL)
file->n_blocks = file->size/BLCKSZ;
/* backup block by block if datafile AND not compressed by cfs*/
if (!backup_data_file(arguments, to_path, file,
arguments->prev_start_lsn,
current.backup_mode,
instance_config.compress_alg,
instance_config.compress_level,
arguments->nodeInfo->checksum_version,
arguments->nodeInfo->ptrack_version_num,
arguments->nodeInfo->ptrack_schema,
true))
{
/* disappeared file not to be confused with 'not changed' */
if (file->write_size != FILE_NOT_FOUND)
file->write_size = BYTES_INVALID;
elog(VERBOSE, "File \"%s\" was not copied to backup", file->path);
continue;
}
}
else if (!file->external_dir_num &&
strcmp(file->name, "pg_control") == 0)
copy_pgcontrol_file(arguments->from_root, FIO_DB_HOST,
arguments->to_root, FIO_BACKUP_HOST,
file);
else
{
const char *dst;
bool skip = false;
char external_dst[MAXPGPATH];
/* If non-data file has not changed since last backup... */
if (prev_file && file->exists_in_prev &&
buf.st_mtime < current.parent_backup)
{
file->crc = pgFileGetCRC(file->path, true, false,
&file->read_size, FIO_DB_HOST);
file->write_size = file->read_size;
/* ...and checksum is the same... */
if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc))
skip = true; /* ...skip copying file. */
}
/* Set file paths */
if (file->external_dir_num)
{
makeExternalDirPathByNum(external_dst,
arguments->external_prefix,
file->external_dir_num);
dst = external_dst;
}
else
dst = arguments->to_root;
if (skip ||
!copy_file(FIO_DB_HOST, dst, FIO_BACKUP_HOST, file, true))
{
/* disappeared file not to be confused with 'not changed' */
if (file->write_size != FILE_NOT_FOUND)
file->write_size = BYTES_INVALID;
elog(VERBOSE, "File \"%s\" was not copied to backup",
file->path);
continue;
}
}
elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes",
file->path, file->write_size);
join_path_components(from_fullpath, arguments->from_root, file->rel_path);
join_path_components(to_fullpath, arguments->to_root, file->rel_path);
}
else
elog(WARNING, "unexpected file type %d", buf.st_mode);
{
char external_dst[MAXPGPATH];
char *external_path = parray_get(arguments->external_dirs,
file->external_dir_num - 1);
makeExternalDirPathByNum(external_dst,
arguments->external_prefix,
file->external_dir_num);
join_path_components(to_fullpath, external_dst, file->rel_path);
join_path_components(from_fullpath, external_path, file->rel_path);
}
/* Encountered some strange beast */
if (!S_ISREG(file->mode))
elog(WARNING, "Unexpected type %d of file \"%s\", skipping",
file->mode, from_fullpath);
/* Check that file exist in previous backup */
if (current.backup_mode != BACKUP_MODE_FULL)
{
pgFile **prev_file_tmp = NULL;
prev_file_tmp = (pgFile **) parray_bsearch(arguments->prev_filelist,
file, pgFileCompareRelPathWithExternal);
if (prev_file_tmp)
{
/* File exists in previous backup */
file->exists_in_prev = true;
prev_file = *prev_file_tmp;
}
}
/* backup file */
if (file->is_datafile && !file->is_cfs)
{
backup_data_file(&(arguments->conn_arg), file, from_fullpath, to_fullpath,
arguments->prev_start_lsn,
current.backup_mode,
instance_config.compress_alg,
instance_config.compress_level,
arguments->nodeInfo->checksum_version,
arguments->nodeInfo->ptrack_version_num,
arguments->nodeInfo->ptrack_schema,
true);
}
else
{
backup_non_data_file(file, prev_file, from_fullpath, to_fullpath,
current.backup_mode, current.parent_backup, true);
}
if (file->write_size == FILE_NOT_FOUND)
continue;
if (file->write_size == BYTES_INVALID)
{
elog(VERBOSE, "Skipping the unchanged file: \"%s\"", from_fullpath);
continue;
}
elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes",
from_fullpath, file->write_size);
}
/* ssh connection to longer needed */

22
src/catalog.c
View File

@ -127,7 +127,7 @@ lock_backup(pgBackup *backup)
pid_t my_pid,
my_p_pid;
pgBackupGetPath(backup, lock_file, lengthof(lock_file), BACKUP_CATALOG_PID);
join_path_components(lock_file, backup->root_dir, BACKUP_CATALOG_PID);
/*
* If the PID in the lockfile is our own PID or our parent's or
@ -674,6 +674,7 @@ pgBackupCreateDir(pgBackup *backup)
elog(ERROR, "backup destination is not empty \"%s\"", path);
fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST);
backup->root_dir = pgut_strdup(path);
/* create directories for actual backup files */
for (i = 0; i < parray_num(subdirs); i++)
@ -1491,6 +1492,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
fio_fprintf(out, "expire-time = '%s'\n", timestamp);
}
if (backup->merge_dest_backup != 0)
fio_fprintf(out, "merge-dest-id = '%s'\n", base36enc(backup->merge_dest_backup));
/*
* Size of PGDATA directory. The size does not include size of related
* WAL segments in archive 'wal' directory.
@ -1699,9 +1703,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
/* use extra variable to avoid reset of previous data_bytes value in case of error */
backup->data_bytes = backup_size_on_disk;
backup->wal_bytes = wal_size_on_disk;
backup->uncompressed_bytes = uncompressed_size_on_disk;
if (backup->stream)
backup->wal_bytes = wal_size_on_disk;
free(buf);
}
@ -1719,6 +1725,7 @@ readBackupControlFile(const char *path)
char *stop_lsn = NULL;
char *status = NULL;
char *parent_backup = NULL;
char *merge_dest_backup = NULL;
char *program_version = NULL;
char *server_version = NULL;
char *compress_alg = NULL;
@ -1748,6 +1755,7 @@ readBackupControlFile(const char *path)
{'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT},
{'s', 0, "status", &status, SOURCE_FILE_STRICT},
{'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT},
{'s', 0, "merge-dest-id", &merge_dest_backup, SOURCE_FILE_STRICT},
{'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT},
{'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT},
{'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT},
@ -1820,6 +1828,8 @@ readBackupControlFile(const char *path)
backup->status = BACKUP_STATUS_RUNNING;
else if (strcmp(status, "MERGING") == 0)
backup->status = BACKUP_STATUS_MERGING;
else if (strcmp(status, "MERGED") == 0)
backup->status = BACKUP_STATUS_MERGED;
else if (strcmp(status, "DELETING") == 0)
backup->status = BACKUP_STATUS_DELETING;
else if (strcmp(status, "DELETED") == 0)
@ -1841,6 +1851,12 @@ readBackupControlFile(const char *path)
free(parent_backup);
}
if (merge_dest_backup)
{
backup->merge_dest_backup = base36dec(merge_dest_backup);
free(merge_dest_backup);
}
if (program_version)
{
StrNCpy(backup->program_version, program_version,
@ -2003,12 +2019,14 @@ pgBackupInit(pgBackup *backup)
backup->stream = false;
backup->from_replica = false;
backup->parent_backup = INVALID_BACKUP_ID;
backup->merge_dest_backup = INVALID_BACKUP_ID;
backup->parent_backup_link = NULL;
backup->primary_conninfo = NULL;
backup->program_version[0] = '\0';
backup->server_version[0] = '\0';
backup->external_dir_str = NULL;
backup->root_dir = NULL;
backup->files = NULL;
}
/* free pgBackup object */

56
src/checkdb.c
View File

@ -39,6 +39,8 @@ typedef struct
ConnectionArgs conn_arg;
/* number of thread for debugging */
int thread_num;
/* pgdata path */
const char *from_root;
/*
* Return value from the thread:
* 0 everything is ok
@ -130,6 +132,7 @@ check_files(void *arg)
int i;
check_files_arg *arguments = (check_files_arg *) arg;
int n_files_list = 0;
char from_fullpath[MAXPGPATH];
if (arguments->files_list)
n_files_list = parray_num(arguments->files_list);
@ -137,49 +140,28 @@ check_files(void *arg)
/* check a file */
for (i = 0; i < n_files_list; i++)
{
int ret;
struct stat buf;
pgFile *file = (pgFile *) parray_get(arguments->files_list, i);
if (!pg_atomic_test_set_flag(&file->lock))
continue;
elog(VERBOSE, "Checking file: \"%s\" ", file->path);
/* check for interrupt */
if (interrupted || thread_interrupted)
elog(ERROR, "interrupted during checkdb");
if (progress)
elog(INFO, "Progress: (%d/%d). Process file \"%s\"",
i + 1, n_files_list, file->path);
/* stat file to check its current state */
ret = stat(file->path, &buf);
if (ret == -1)
{
if (errno == ENOENT)
{
/*
* If file is not found, this is not en error.
* It could have been deleted by concurrent postgres transaction.
*/
elog(LOG, "File \"%s\" is not found", file->path);
continue;
}
else
{
elog(ERROR,
"can't stat file to check \"%s\": %s",
file->path, strerror(errno));
}
}
/* No need to check directories */
if (S_ISDIR(buf.st_mode))
if (S_ISDIR(file->mode))
continue;
if (S_ISREG(buf.st_mode))
if (!pg_atomic_test_set_flag(&file->lock))
continue;
join_path_components(from_fullpath, arguments->from_root, file->rel_path);
elog(VERBOSE, "Checking file: \"%s\" ", from_fullpath);
if (progress)
elog(INFO, "Progress: (%d/%d). Process file \"%s\"",
i + 1, n_files_list, from_fullpath);
if (S_ISREG(file->mode))
{
/* check only uncompressed by cfs datafiles */
if (file->is_datafile && !file->is_cfs)
@ -189,13 +171,14 @@ check_files(void *arg)
* uses global variables to set connections.
* Need refactoring.
*/
if (!check_data_file(&(arguments->conn_arg), file,
if (!check_data_file(&(arguments->conn_arg),
file, from_fullpath,
arguments->checksum_version))
arguments->ret = 2; /* corruption found */
}
}
else
elog(WARNING, "unexpected file type %d", buf.st_mode);
elog(WARNING, "unexpected file type %d", file->mode);
}
/* Ret values:
@ -258,6 +241,7 @@ do_block_validation(char *pgdata, uint32 checksum_version)
arg->files_list = files_list;
arg->checksum_version = checksum_version;
arg->from_root = pgdata;
arg->conn_arg.conn = NULL;
arg->conn_arg.cancel_conn = NULL;

847
src/data.c
View File

File diff suppressed because it is too large Load Diff

68
src/delete.c
View File

@ -482,14 +482,14 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l
*/
keep_backup_id = base36enc_dup(keep_backup->start_time);
elog(INFO, "Merge incremental chain between FULL backup %s and backup %s",
elog(INFO, "Merge incremental chain between full backup %s and backup %s",
base36enc(full_backup->start_time), keep_backup_id);
pg_free(keep_backup_id);
merge_list = parray_new();
/* Form up a merge list */
while(keep_backup->parent_backup_link)
while (keep_backup->parent_backup_link)
{
parray_append(merge_list, keep_backup);
keep_backup = keep_backup->parent_backup_link;
@ -515,6 +515,19 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l
/* Lock merge chain */
catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0);
/* Consider this extreme case */
// PAGEa1 PAGEb1 both valid
// \ /
// FULL
/* Check that FULL backup do not has multiple descendants
* full_backup always point to current full_backup after merge
*/
// if (is_prolific(backup_list, full_backup))
// {
// elog(WARNING, "Backup %s has multiple valid descendants. "
// "Automatic merge is not possible.", base36enc(full_backup->start_time));
// }
/* Merge list example:
* 0 PAGE3
@ -522,38 +535,26 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l
* 2 PAGE1
* 3 FULL
*
* Consequentially merge incremental backups from PAGE1 to PAGE3
* into FULL.
* Merge incremental chain from PAGE3 into FULL.
*/
keep_backup = parray_get(merge_list, 0);
merge_chain(merge_list, full_backup, keep_backup);
backup_merged = true;
for (j = parray_num(merge_list) - 2; j >= 0; j--)
{
pgBackup *from_backup = (pgBackup *) parray_get(merge_list, j);
/* Consider this extreme case */
// PAGEa1 PAGEb1 both valid
// \ /
// FULL
/* Check that FULL backup do not has multiple descendants
* full_backup always point to current full_backup after merge
*/
if (is_prolific(backup_list, full_backup))
{
elog(WARNING, "Backup %s has multiple valid descendants. "
"Automatic merge is not possible.", base36enc(full_backup->start_time));
break;
}
merge_backups(full_backup, from_backup);
backup_merged = true;
pgBackup *tmp_backup = (pgBackup *) parray_get(merge_list, j);
/* Try to remove merged incremental backup from both keep and purge lists */
parray_rm(to_purge_list, from_backup, pgBackupCompareId);
parray_rm(to_purge_list, tmp_backup, pgBackupCompareId);
parray_set(to_keep_list, i, NULL);
}
pgBackupValidate(full_backup, NULL);
if (full_backup->status == BACKUP_STATUS_CORRUPT)
elog(ERROR, "Merging of backup %s failed", base36enc(full_backup->start_time));
/* Cleanup */
parray_free(merge_list);
}
@ -724,10 +725,10 @@ void
delete_backup_files(pgBackup *backup)
{
size_t i;
char path[MAXPGPATH];
char timestamp[100];
parray *files;
parray *files;
size_t num_files;
char full_path[MAXPGPATH];
/*
* If the backup was deleted already, there is nothing to do.
@ -752,8 +753,7 @@ delete_backup_files(pgBackup *backup)
/* list files to be deleted */
files = parray_new();
pgBackupGetPath(backup, path, lengthof(path), NULL);
dir_list_file(files, path, false, true, true, 0, FIO_BACKUP_HOST);
dir_list_file(files, backup->root_dir, false, true, true, 0, FIO_BACKUP_HOST);
/* delete leaf node first */
parray_qsort(files, pgFileComparePathDesc);
@ -762,14 +762,16 @@ delete_backup_files(pgBackup *backup)
{
pgFile *file = (pgFile *) parray_get(files, i);
if (progress)
elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"",
i + 1, num_files, file->path);
join_path_components(full_path, backup->root_dir, file->rel_path);
if (interrupted)
elog(ERROR, "interrupted during delete backup");
pgFileDelete(file);
if (progress)
elog(INFO, "Progress: (%zd/%zd). Delete file \"%s\"",
i + 1, num_files, full_path);
pgFileDelete(file, full_path);
}
parray_walk(files, pgFileFree);

103
src/dir.c
View File

@ -181,6 +181,7 @@ pgFileNew(const char *path, const char *rel_path, bool follow_symlink,
file = pgFileInit(path, rel_path);
file->size = st.st_size;
file->mode = st.st_mode;
file->mtime = st.st_mtime;
file->external_dir_num = external_dir_num;
return file;
@ -226,33 +227,95 @@ pgFileInit(const char *path, const char *rel_path)
* If the pgFile points directory, the directory must be empty.
*/
void
pgFileDelete(pgFile *file)
pgFileDelete(pgFile *file, const char *full_path)
{
if (S_ISDIR(file->mode))
{
if (rmdir(file->path) == -1)
if (rmdir(full_path) == -1)
{
if (errno == ENOENT)
return;
else if (errno == ENOTDIR) /* could be symbolic link */
goto delete_file;
elog(ERROR, "cannot remove directory \"%s\": %s",
file->path, strerror(errno));
elog(ERROR, "Cannot remove directory \"%s\": %s",
full_path, strerror(errno));
}
return;
}
delete_file:
if (remove(file->path) == -1)
if (remove(full_path) == -1)
{
if (errno == ENOENT)
return;
elog(ERROR, "cannot remove file \"%s\": %s", file->path,
elog(ERROR, "Cannot remove file \"%s\": %s", full_path,
strerror(errno));
}
}
/*
* Read the local file to compute its CRC.
* We cannot make decision about file decompression because
* user may ask to backup already compressed files and we should be
* obvious about it.
* TODO: add decompression option.
*/
pg_crc32
pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok)
{
FILE *fp;
pg_crc32 crc = 0;
char buf[STDIO_BUFSIZE];
size_t len = 0;
INIT_FILE_CRC32(use_crc32c, crc);
/* open file in binary read mode */
fp = fopen(file_path, PG_BINARY_R);
if (fp == NULL)
{
if (errno == ENOENT)
{
if (missing_ok)
{
FIN_FILE_CRC32(use_crc32c, crc);
return crc;
}
}
elog(ERROR, "Cannot open file \"%s\": %s",
file_path, strerror(errno));
}
/* calc CRC of file */
for (;;)
{
if (interrupted)
elog(ERROR, "interrupted during CRC calculation");
len = fread(&buf, 1, sizeof(buf), fp);
if (len == 0)
{
/* we either run into eof or error */
if (feof(fp))
break;
if (ferror(fp))
elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
}
/* update CRC */
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
}
FIN_FILE_CRC32(use_crc32c, crc);
fclose(fp);
return crc;
}
/*
* Read the file to compute its CRC.
* As a handy side effect, we return filesize via bytes_read parameter.
@ -263,7 +326,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted,
{
FILE *fp;
pg_crc32 crc = 0;
char buf[1024];
char buf[STDIO_BUFSIZE];
size_t len = 0;
size_t total = 0;
int errno_tmp;
@ -291,7 +354,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted,
elog(ERROR, "interrupted during CRC calculation");
len = fio_fread(fp, buf, sizeof(buf));
if(len == 0)
if (len == 0)
break;
/* update CRC */
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
@ -1150,7 +1213,7 @@ read_tablespace_map(parray *files, const char *backup_dir)
void
check_tablespace_mapping(pgBackup *backup)
{
char this_backup_path[MAXPGPATH];
// char this_backup_path[MAXPGPATH];
parray *links;
size_t i;
TablespaceListCell *cell;
@ -1158,8 +1221,8 @@ check_tablespace_mapping(pgBackup *backup)
links = parray_new();
pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL);
read_tablespace_map(links, this_backup_path);
// pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL);
read_tablespace_map(links, backup->root_dir);
/* Sort links by the path of a linked file*/
parray_qsort(links, pgFileCompareLinked);
@ -1687,15 +1750,16 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
{
FILE *fp;
pgFile *file;
char path[MAXPGPATH];
char database_dir[MAXPGPATH];
char database_map_path[MAXPGPATH];
pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR);
join_path_components(database_map_path, path, DATABASE_MAP);
// pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR);
join_path_components(database_dir, backup->root_dir, DATABASE_DIR);
join_path_components(database_map_path, database_dir, DATABASE_MAP);
fp = fio_fopen(database_map_path, PG_BINARY_W, FIO_BACKUP_HOST);
if (fp == NULL)
elog(ERROR, "Cannot open database map \"%s\": %s", path,
elog(ERROR, "Cannot open database map \"%s\": %s", database_map_path,
strerror(errno));
print_database_map(fp, database_map);
@ -1710,9 +1774,9 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
file = pgFileNew(database_map_path, DATABASE_MAP, true, 0,
FIO_BACKUP_HOST);
pfree(file->path);
file->path = strdup(DATABASE_MAP);
file->crc = pgFileGetCRC(database_map_path, true, false,
&file->read_size, FIO_BACKUP_HOST);
file->path = pgut_strdup(DATABASE_MAP);
file->crc = pgFileGetCRCnew(database_map_path, true, false);
file->write_size = file->read_size;
file->uncompressed_size = file->read_size;
parray_append(backup_files_list, file);
@ -1730,7 +1794,8 @@ read_database_map(pgBackup *backup)
char path[MAXPGPATH];
char database_map_path[MAXPGPATH];
pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR);
// pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR);
join_path_components(path, backup->root_dir, DATABASE_DIR);
join_path_components(database_map_path, path, DATABASE_MAP);
fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST);

1312
src/merge.c
View File

File diff suppressed because it is too large Load Diff

2
src/pg_probackup.c
View File

@ -754,7 +754,7 @@ main(int argc, char *argv[])
elog(ERROR, "required parameter not specified: BACKUP_MODE "
"(-b, --backup-mode)");
return do_backup(start_time, no_validate, set_backup_params);
return do_backup(start_time, no_validate, set_backup_params, no_sync);
}
case RESTORE_CMD:
return do_restore_or_validate(current.backup_id,

67
src/pg_probackup.h
View File

@ -85,6 +85,9 @@ extern const char *PROGRAM_EMAIL;
#define STDOUT_FILENO 1
#endif
/* stdio buffer size */
#define STDIO_BUFSIZE 65536
/* Check if an XLogRecPtr value is pointed to 0 offset */
#define XRecOffIsNull(xlrp) \
((xlrp) % XLOG_BLCKSZ == 0)
@ -139,6 +142,8 @@ typedef struct pgFile
char *name; /* file or directory name */
mode_t mode; /* protection (file type and permission) */
size_t size; /* size of the file */
time_t mtime; /* file st_mtime attribute, can be used only
during backup */
size_t read_size; /* size of the portion read (if only some pages are
backed up, it's different from size) */
int64 write_size; /* size of the backed-up file. BYTES_INVALID means
@ -188,6 +193,8 @@ typedef enum BackupStatus
BACKUP_STATUS_ERROR, /* aborted because of unexpected error */
BACKUP_STATUS_RUNNING, /* running backup */
BACKUP_STATUS_MERGING, /* merging backups */
BACKUP_STATUS_MERGED, /* backup has been successfully merged and now awaits
* the assignment of new start_time */
BACKUP_STATUS_DELETING, /* data files are being deleted */
BACKUP_STATUS_DELETED, /* data files have been deleted */
BACKUP_STATUS_DONE, /* completed but not validated yet */
@ -322,6 +329,10 @@ struct pgBackup
XLogRecPtr stop_lsn; /* backup's finishing transaction log location */
time_t start_time; /* since this moment backup has status
* BACKUP_STATUS_RUNNING */
time_t merge_dest_backup; /* start_time of incremental backup,
* this backup is merging with.
* Only available for FULL backups
* with MERGING or MERGED statuses */
time_t merge_time; /* the moment when merge was started or 0 */
time_t end_time; /* the moment when backup was finished, or the moment
* when we realized that backup is broken */
@ -372,10 +383,10 @@ struct pgBackup
* in the format suitable for recovery.conf */
char *external_dir_str; /* List of external directories,
* separated by ':' */
parray *files; /* list of files belonging to this backup
* must be populated by calling backup_populate() */
char *root_dir; /* Full path for root backup directory:
backup_path/instance_name/backup_id */
parray *files; /* list of files belonging to this backup
* must be populated explicitly */
};
/* Recovery target for restore and validate subcommands */
@ -513,6 +524,7 @@ typedef struct BackupPageHeader
} BackupPageHeader;
/* Special value for compressed_size field */
#define PageIsOk 0
#define PageIsTruncated -2
#define SkipCurrentPage -3
#define PageIsCorrupted -4 /* used by checkdb */
@ -629,7 +641,7 @@ extern const char *pgdata_exclude_dir[];
/* in backup.c */
extern int do_backup(time_t start_time, bool no_validate,
pgSetBackupParams *set_backup_params);
pgSetBackupParams *set_backup_params, bool no_sync);
extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt,
char *pgdata);
extern BackupMode parse_backup_mode(const char *value);
@ -664,6 +676,8 @@ extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetT
/* in merge.c */
extern void do_merge(time_t backup_id);
extern void merge_backups(pgBackup *backup, pgBackup *next_backup);
extern void merge_chain(parray *parent_chain,
pgBackup *full_backup, pgBackup *dest_backup);
extern parray *read_database_map(pgBackup *backup);
@ -698,6 +712,9 @@ extern char *slurpFile(const char *datadir,
size_t *filesize,
bool safe,
fio_location location);
extern char *slurpFileFullPath(const char *from_fullpath,
size_t *filesize, bool safe,
fio_location location);
extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize);
/* in help.c */
@ -809,10 +826,13 @@ extern pgFile *pgFileNew(const char *path, const char *rel_path,
bool follow_symlink, int external_dir_num,
fio_location location);
extern pgFile *pgFileInit(const char *path, const char *rel_path);
extern void pgFileDelete(pgFile *file);
extern void pgFileDelete(pgFile *file, const char *full_path);
extern void pgFileFree(void *file);
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c,
bool raise_on_deleted, size_t *bytes_read, fio_location location);
extern pg_crc32 pgFileGetCRCnew(const char *file_path, bool missing_ok, bool use_crc32c);
//extern pg_crc32 pgFileGetCRC_compressed(const char *file_path, bool use_crc32c, bool missing_ok);
extern int pgFileCompareName(const void *f1, const void *f2);
extern int pgFileComparePath(const void *f1, const void *f2);
extern int pgFileMapComparePath(const void *f1, const void *f2);
@ -826,21 +846,24 @@ extern int pgFileCompareSize(const void *f1, const void *f2);
extern int pgCompareOid(const void *f1, const void *f2);
/* in data.c */
extern bool check_data_file(ConnectionArgs* arguments, pgFile* file, uint32 checksum_version);
extern bool backup_data_file(backup_files_arg* arguments,
const char *to_path, pgFile *file,
XLogRecPtr prev_backup_start_lsn,
BackupMode backup_mode,
CompressAlg calg, int clevel,
uint32 checksum_version,
int ptrack_version_num,
const char *ptrack_schema,
bool missing_ok);
extern void restore_data_file(const char *to_path,
pgFile *file, bool allow_truncate,
bool write_header,
uint32 backup_version);
extern size_t restore_data_file_new(parray *parent_chain, pgFile *dest_file,
extern bool check_data_file(ConnectionArgs *arguments, pgFile *file,
const char *from_fullpath, uint32 checksum_version);
extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
const char *from_fullpath, const char *to_fullpath,
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
CompressAlg calg, int clevel, uint32 checksum_version,
int ptrack_version_num, const char *ptrack_schema, bool missing_ok);
extern void backup_non_data_file(pgFile *file, pgFile *prev_file,
const char *from_fullpath, const char *to_fullpath,
BackupMode backup_mode, time_t parent_backup_time,
bool missing_ok);
extern void backup_non_data_file_internal(const char *from_fullpath,
fio_location from_location,
const char *to_fullpath, pgFile *file,
bool missing_ok);
extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file,
FILE *out, const char *to_fullpath);
extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
const char *from_fullpath, const char *to_fullpath, int nblocks);
@ -848,8 +871,6 @@ extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
pgFile *dest_file, FILE *out, const char *to_fullpath);
extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
const char *from_fullpath, const char *to_fullpath);
extern bool copy_file(fio_location from_location, const char *to_root,
fio_location to_location, pgFile *file, bool missing_ok);
extern bool create_empty_file(fio_location from_location, const char *to_root,
fio_location to_location, pgFile *file);
@ -887,8 +908,8 @@ extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);
extern uint32 get_xlog_seg_size(char *pgdata_path);
extern void set_min_recovery_point(pgFile *file, const char *backup_path,
XLogRecPtr stop_backup_lsn);
extern void copy_pgcontrol_file(const char *from_root, fio_location location, const char *to_root, fio_location to_location,
pgFile *file);
extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location,
const char *to_fullpath, fio_location to_location, pgFile *file);
extern void time2iso(char *buf, size_t len, time_t time);
extern const char *status2str(BackupStatus status);

346
src/restore.c
View File

@ -17,24 +17,6 @@
#include "utils/thread.h"
typedef struct
{
parray *files;
pgBackup *backup;
parray *external_dirs;
char *external_prefix;
parray *dest_external_dirs;
parray *dest_files;
parray *dbOid_exclude_list;
bool skip_external_dirs;
/*
* Return value from the thread.
* 0 means there is no error, 1 - there is an error.
*/
int ret;
} restore_files_arg;
typedef struct
{
parray *dest_files;
@ -51,11 +33,8 @@ typedef struct
* 0 means there is no error, 1 - there is an error.
*/
int ret;
} restore_files_arg_new;
} restore_files_arg;
static void restore_backup(pgBackup *backup, parray *dest_external_dirs,
parray *dest_files, parray *dbOid_exclude_list,
pgRestoreParams *params);
static void create_recovery_conf(time_t backup_id,
pgRecoveryTarget *rt,
pgBackup *backup,
@ -68,9 +47,6 @@ static void restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray *dbOid_exclude_list, pgRestoreParams *params,
const char *pgdata_path, bool no_sync);
static void *restore_files_new(void *arg);
/*
* Iterate over backup list to find all ancestors of the broken parent_backup
* and update their status to BACKUP_STATUS_ORPHAN
@ -522,7 +498,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray *external_dirs = NULL;
/* arrays with meta info for multi threaded backup */
pthread_t *threads;
restore_files_arg_new *threads_args;
restore_files_arg *threads_args;
bool restore_isok = true;
/* fancy reporting */
@ -644,7 +620,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
fio_disconnect();
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
threads_args = (restore_files_arg_new *) palloc(sizeof(restore_files_arg_new) *
threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) *
num_threads);
/* Restore files into target directory */
@ -659,7 +635,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
thread_interrupted = false;
for (i = 0; i < num_threads; i++)
{
restore_files_arg_new *arg = &(threads_args[i]);
restore_files_arg *arg = &(threads_args[i]);
arg->dest_files = dest_files;
arg->dest_backup = dest_backup;
@ -675,7 +651,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
/* Useless message TODO: rewrite */
elog(LOG, "Start thread %i", i + 1);
pthread_create(&threads[i], NULL, restore_files_new, arg);
pthread_create(&threads[i], NULL, restore_files, arg);
}
/* Wait theads */
@ -767,14 +743,14 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
* Restore files into $PGDATA.
*/
static void *
restore_files_new(void *arg)
restore_files(void *arg)
{
int i;
char to_fullpath[MAXPGPATH];
FILE *out = NULL;
char buffer[65536];
char buffer[STDIO_BUFSIZE];
restore_files_arg_new *arguments = (restore_files_arg_new *) arg;
restore_files_arg *arguments = (restore_files_arg *) arg;
for (i = 0; i < parray_num(arguments->dest_files); i++)
{
@ -870,8 +846,8 @@ restore_files_new(void *arg)
/* Restore destination file */
if (dest_file->is_datafile && !dest_file->is_cfs)
/* Destination file is data file */
arguments->restored_bytes += restore_data_file_new(arguments->parent_chain,
dest_file, out, to_fullpath);
arguments->restored_bytes += restore_data_file(arguments->parent_chain,
dest_file, out, to_fullpath);
else
/* Destination file is non-data file */
arguments->restored_bytes += restore_non_data_file(arguments->parent_chain,
@ -885,7 +861,7 @@ restore_files_new(void *arg)
done:
/* truncate file up to n_blocks. NOTE: no need, we just should not write
/* Truncate file up to n_blocks. NOTE: no need, we just should not write
* blocks that are exceeding n_blocks.
* But for this to work, n_blocks should be trusted.
*/
@ -916,306 +892,6 @@ restore_files_new(void *arg)
return NULL;
}
/*
* Restore one backup.
*/
void
restore_backup(pgBackup *backup, parray *dest_external_dirs,
parray *dest_files, parray *dbOid_exclude_list,
pgRestoreParams *params)
{
char timestamp[100];
char database_path[MAXPGPATH];
char external_prefix[MAXPGPATH];
char list_path[MAXPGPATH];
parray *files;
parray *external_dirs = NULL;
int i;
/* arrays with meta info for multi threaded backup */
pthread_t *threads;
restore_files_arg *threads_args;
bool restore_isok = true;
if (backup->status != BACKUP_STATUS_OK &&
backup->status != BACKUP_STATUS_DONE)
{
if (params->force)
elog(WARNING, "Backup %s is not valid, restore is forced",
base36enc(backup->start_time));
else
elog(ERROR, "Backup %s cannot be restored because it is not valid",
base36enc(backup->start_time));
}
/* confirm block size compatibility */
if (backup->block_size != BLCKSZ)
elog(ERROR,
"BLCKSZ(%d) is not compatible(%d expected)",
backup->block_size, BLCKSZ);
if (backup->wal_block_size != XLOG_BLCKSZ)
elog(ERROR,
"XLOG_BLCKSZ(%d) is not compatible(%d expected)",
backup->wal_block_size, XLOG_BLCKSZ);
time2iso(timestamp, lengthof(timestamp), backup->start_time);
elog(LOG, "Restoring database from backup %s", timestamp);
if (backup->external_dir_str)
external_dirs = make_external_directory_list(backup->external_dir_str,
true);
/*
* Get list of files which need to be restored.
*/
pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR);
pgBackupGetPath(backup, external_prefix, lengthof(external_prefix),
EXTERNAL_DIR);
pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST);
files = dir_read_file_list(database_path, external_prefix, list_path,
FIO_BACKUP_HOST);
/* Restore directories in do_backup_instance way */
parray_qsort(files, pgFileComparePath);
/*
* Make external directories before restore
* and setup threads at the same time
*/
for (i = 0; i < parray_num(files); i++)
{
pgFile *file = (pgFile *) parray_get(files, i);
/*
* If the entry was an external directory, create it in the backup.
*/
if (!params->skip_external_dirs &&
file->external_dir_num && S_ISDIR(file->mode) &&
/* Do not create unnecessary external directories */
parray_bsearch(dest_files, file, pgFileCompareRelPathWithExternal))
{
char *external_path;
if (!external_dirs ||
parray_num(external_dirs) < file->external_dir_num - 1)
elog(ERROR, "Inconsistent external directory backup metadata");
external_path = parray_get(external_dirs,
file->external_dir_num - 1);
if (backup_contains_external(external_path, dest_external_dirs))
{
char container_dir[MAXPGPATH];
char dirpath[MAXPGPATH];
char *dir_name;
makeExternalDirPathByNum(container_dir, external_prefix,
file->external_dir_num);
dir_name = GetRelativePath(file->path, container_dir);
elog(VERBOSE, "Create directory \"%s\"", dir_name);
join_path_components(dirpath, external_path, dir_name);
fio_mkdir(dirpath, DIR_PERMISSION, FIO_DB_HOST);
}
}
/* setup threads */
pg_atomic_clear_flag(&file->lock);
}
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) *
num_threads);
/* Restore files into target directory */
thread_interrupted = false;
for (i = 0; i < num_threads; i++)
{
restore_files_arg *arg = &(threads_args[i]);
arg->files = files;
arg->backup = backup;
arg->external_dirs = external_dirs;
arg->external_prefix = external_prefix;
arg->dest_external_dirs = dest_external_dirs;
arg->dest_files = dest_files;
arg->dbOid_exclude_list = dbOid_exclude_list;
arg->skip_external_dirs = params->skip_external_dirs;
/* By default there are some error */
threads_args[i].ret = 1;
/* Useless message TODO: rewrite */
elog(LOG, "Start thread for num:%zu", parray_num(files));
pthread_create(&threads[i], NULL, restore_files, arg);
}
/* Wait theads */
for (i = 0; i < num_threads; i++)
{
pthread_join(threads[i], NULL);
if (threads_args[i].ret == 1)
restore_isok = false;
}
if (!restore_isok)
elog(ERROR, "Data files restoring failed");
pfree(threads);
pfree(threads_args);
/* cleanup */
parray_walk(files, pgFileFree);
parray_free(files);
if (external_dirs != NULL)
free_dir_list(external_dirs);
elog(LOG, "Restore %s backup completed", base36enc(backup->start_time));
}
/*
* Restore files into $PGDATA.
*/
static void *
restore_files(void *arg)
{
int i;
restore_files_arg *arguments = (restore_files_arg *)arg;
for (i = 0; i < parray_num(arguments->files); i++)
{
char from_root[MAXPGPATH];
pgFile *file = (pgFile *) parray_get(arguments->files, i);
if (!pg_atomic_test_set_flag(&file->lock))
continue;
pgBackupGetPath(arguments->backup, from_root,
lengthof(from_root), DATABASE_DIR);
/* check for interrupt */
if (interrupted || thread_interrupted)
elog(ERROR, "Interrupted during restore database");
/* Directories were created before */
if (S_ISDIR(file->mode))
continue;
if (progress)
elog(INFO, "Progress: (%d/%lu). Process file %s ",
i + 1, (unsigned long) parray_num(arguments->files),
file->rel_path);
/* Only files from pgdata can be skipped by partial restore */
if (arguments->dbOid_exclude_list && file->external_dir_num == 0)
{
/* Check if the file belongs to the database we exclude */
if (parray_bsearch(arguments->dbOid_exclude_list,
&file->dbOid, pgCompareOid))
{
/*
* We cannot simply skip the file, because it may lead to
* failure during WAL redo; hence, create empty file.
*/
create_empty_file(FIO_BACKUP_HOST,
instance_config.pgdata, FIO_DB_HOST, file);
elog(VERBOSE, "Exclude file due to partial restore: \"%s\"",
file->rel_path);
continue;
}
}
/*
* For PAGE and PTRACK backups skip datafiles which haven't changed
* since previous backup and thus were not backed up.
* We cannot do the same when restoring DELTA backup because we need information
* about every datafile to correctly truncate them.
*/
if (file->write_size == BYTES_INVALID)
{
/* data file, only PAGE and PTRACK can skip */
if (((file->is_datafile && !file->is_cfs) &&
(arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) ||
/* non-data file can be skipped regardless of backup type */
!(file->is_datafile && !file->is_cfs))
{
elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", file->path);
continue;
}
}
/* Do not restore tablespace_map file */
if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path))
{
elog(VERBOSE, "Skip tablespace_map");
continue;
}
/* Do not restore database_map file */
if ((file->external_dir_num == 0) &&
strcmp(DATABASE_MAP, file->rel_path) == 0)
{
elog(VERBOSE, "Skip database_map");
continue;
}
/* Do no restore external directory file if a user doesn't want */
if (arguments->skip_external_dirs && file->external_dir_num > 0)
continue;
/* Skip unnecessary file */
if (parray_bsearch(arguments->dest_files, file,
pgFileCompareRelPathWithExternal) == NULL)
continue;
/*
* restore the file.
* We treat datafiles separately, cause they were backed up block by
* block and have BackupPageHeader meta information, so we cannot just
* copy the file from backup.
*/
elog(VERBOSE, "Restoring file \"%s\", is_datafile %i, is_cfs %i",
file->path, file->is_datafile?1:0, file->is_cfs?1:0);
if (file->is_datafile && !file->is_cfs)
{
char to_path[MAXPGPATH];
join_path_components(to_path, instance_config.pgdata,
file->rel_path);
restore_data_file(to_path, file,
arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA,
false,
parse_program_version(arguments->backup->program_version));
}
else if (file->external_dir_num)
{
char *external_path = parray_get(arguments->external_dirs,
file->external_dir_num - 1);
if (backup_contains_external(external_path,
arguments->dest_external_dirs))
copy_file(FIO_BACKUP_HOST,
external_path, FIO_DB_HOST, file, false);
}
else if (strcmp(file->name, "pg_control") == 0)
copy_pgcontrol_file(from_root, FIO_BACKUP_HOST,
instance_config.pgdata, FIO_DB_HOST,
file);
else
copy_file(FIO_BACKUP_HOST,
instance_config.pgdata, FIO_DB_HOST,
file, false);
/* print size of restored file */
if (file->write_size != BYTES_INVALID)
elog(VERBOSE, "Restored file %s : " INT64_FORMAT " bytes",
file->path, file->write_size);
}
/* Data files restoring is successful */
arguments->ret = 0;
return NULL;
}
/*
* Create recovery.conf (probackup_recovery.conf in case of PG12)
* with given recovery target parameters

15
src/util.c
View File

@ -109,7 +109,7 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
* Write ControlFile to pg_control
*/
static void
writeControlFile(ControlFileData *ControlFile, char *path, fio_location location)
writeControlFile(ControlFileData *ControlFile, const char *path, fio_location location)
{
int fd;
char *buffer = NULL;
@ -135,7 +135,7 @@ writeControlFile(ControlFileData *ControlFile, char *path, fio_location location
elog(ERROR, "Failed to overwrite file: %s", path);
if (fio_flush(fd) != 0)
elog(ERROR, "Failed to fsync file: %s", path);
elog(ERROR, "Failed to sync file: %s", path);
fio_close(fd);
pg_free(buffer);
@ -383,15 +383,14 @@ set_min_recovery_point(pgFile *file, const char *backup_path,
* Copy pg_control file to backup. We do not apply compression to this file.
*/
void
copy_pgcontrol_file(const char *from_root, fio_location from_location,
const char *to_root, fio_location to_location, pgFile *file)
copy_pgcontrol_file(const char *from_fullpath, fio_location from_location,
const char *to_fullpath, fio_location to_location, pgFile *file)
{
ControlFileData ControlFile;
char *buffer;
size_t size;
char to_path[MAXPGPATH];
buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false, from_location);
buffer = slurpFileFullPath(from_fullpath, &size, false, from_location);
digestControlFile(&ControlFile, buffer, size);
@ -400,8 +399,7 @@ copy_pgcontrol_file(const char *from_root, fio_location from_location,
file->write_size = size;
file->uncompressed_size = size;
join_path_components(to_path, to_root, file->rel_path);
writeControlFile(&ControlFile, to_path, to_location);
writeControlFile(&ControlFile, to_fullpath, to_location);
pg_free(buffer);
}
@ -471,6 +469,7 @@ status2str(BackupStatus status)
"ERROR",
"RUNNING",
"MERGING",
"MERGED",
"DELETING",
"DELETED",
"DONE",