db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-25 09:01:48 +02:00
Code Issues Releases Activity

[Issue #169] Refactoring, added support for external directories and bugfixes

Browse Source
This commit is contained in:
Grigory Smolkin 2020-01-26 03:41:55 +03:00
parent 7f983c599a
commit 577e8763bc
6 changed files with 430 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/catalog.c
View File

@ -444,6 +444,9 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id)
base36enc(backup->start_time), backup_conf_path);
}
backup->root_dir = pgut_strdup(data_path);
/* TODO: save encoded backup id */
backup->backup_id = backup->start_time;
if (requested_backup_id != INVALID_BACKUP_ID
&& requested_backup_id != backup->start_time)
@ -2005,6 +2008,7 @@ pgBackupInit(pgBackup *backup)
backup->program_version[0] = '\0';
backup->server_version[0] = '\0';
backup->external_dir_str = NULL;
backup->root_dir = NULL;
}
/* free pgBackup object */
@ -2015,6 +2019,7 @@ pgBackupFree(void *backup)
pfree(b->primary_conninfo);
pfree(b->external_dir_str);
pfree(b->root_dir);
pfree(backup);
}

217
src/data.c
View File

@ -962,23 +962,88 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate,
}
/*
* Restore files in the from_root directory to the to_root directory with
* same relative path.
*
* If write_header is true then we add header to each restored block, currently
* it is used for MERGE command.
*
* to_fullpath and from_fullpath are provided strictly for ERROR reporting
* Iterate over parent backup chain and lookup given destination file in
* filelist of every chain member starting with FULL backup.
* Apply changed blocks to destination file from every backup in parent chain.
*/
void
restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath)
{
int i;
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
{
char from_root[MAXPGPATH];
char from_fullpath[MAXPGPATH];
FILE *in = NULL;
pgFile **res_file = NULL;
pgFile *tmp_file = NULL;
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
/* check for interrupt */
if (interrupted || thread_interrupted)
elog(ERROR, "Interrupted during restore");
/* lookup file in intermediate backup */
res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal);
tmp_file = (res_file) ? *res_file : NULL;
/* Destination file is not exists yet at this moment */
if (tmp_file == NULL)
continue;
/*
* Skip file if it haven't changed since previous backup
* and thus was not backed up.
*/
if (tmp_file->write_size == BYTES_INVALID)
{
// elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", tmp_file->rel_path);
continue;
}
/*
* At this point we are sure, that something is going to be copied
* Open source file.
*/
join_path_components(from_root, backup->root_dir, DATABASE_DIR);
join_path_components(from_fullpath, from_root, tmp_file->rel_path);
in = fopen(from_fullpath, PG_BINARY_R);
if (in == NULL)
{
elog(INFO, "Cannot open backup file \"%s\": %s", from_fullpath,
strerror(errno));
Assert(0);
}
/*
* restore the file.
* Datafiles are backed up block by block and every block
* have BackupPageHeader with meta information, so we cannot just
* copy the file from backup.
*/
restore_data_file_internal(in, out, tmp_file,
parse_program_version(backup->program_version),
from_fullpath, to_fullpath, dest_file->n_blocks);
if (fio_fclose(in) != 0)
elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
strerror(errno));
}
}
void
restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
const char *from_fullpath, const char *to_fullpath, int nblocks)
{
BackupPageHeader header;
BlockNumber blknum = 0;
size_t write_len = 0;
while (true)
for (;;)
{
off_t write_pos;
size_t read_len;
@ -1016,9 +1081,44 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
blknum = header.block;
/*
* Backupward compatibility kludge: in the good old days
* n_blocks attribute was available only in DELTA backups.
* File truncate in PAGE and PTRACK happened on the fly when
* special value PageIsTruncated is encountered.
* It is inefficient.
*
* Nowadays every backup type has n_blocks, so instead
* writing and then truncating redundant data, writing
* is not happening in the first place.
* TODO: remove in 3.0.0
*/
if (header.compressed_size == PageIsTruncated)
{
/*
* Block header contains information that this block was truncated.
* We need to truncate file to this length.
*/
elog(VERBOSE, "Truncate file \"%s\" to block %u", to_fullpath, header.block);
/* To correctly truncate file, we must first flush STDIO buffers */
if (fio_fflush(out) != 0)
elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno));
/* Set position to the start of file */
if (fio_fseek(out, 0) < 0)
elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno));
if (fio_ftruncate(out, header.block * BLCKSZ) != 0)
elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno));
break;
}
/* no point in writing redundant data */
if (nblocks > 0 && blknum >= nblocks)
return;
break;
if (header.compressed_size > BLCKSZ)
elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum);
@ -1061,7 +1161,6 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
/*
* Seek and write the restored page.
* TODO: invent fio_pwrite().
*/
if (fio_fseek(out, write_pos) < 0)
elog(ERROR, "Cannot seek block %u of \"%s\": %s",
@ -1096,7 +1195,7 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
* it is either small control file or already compressed cfs file.
*/
void
restore_non_data_file(FILE *in, FILE *out, pgFile *file,
restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
const char *from_fullpath, const char *to_fullpath)
{
size_t read_len = 0;
@ -1148,6 +1247,100 @@ restore_non_data_file(FILE *in, FILE *out, pgFile *file,
elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size);
}
void
restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
pgFile *dest_file, FILE *out, const char *to_fullpath)
{
int i;
char from_root[MAXPGPATH];
char from_fullpath[MAXPGPATH];
FILE *in = NULL;
pgFile *tmp_file = NULL;
pgBackup *tmp_backup = NULL;
/* Check if full copy of destination file is available in destination backup */
if (dest_file->write_size > 0)
{
tmp_file = dest_file;
tmp_backup = dest_backup;
}
else
{
/*
* Iterate over parent chain starting from direct parent of destination
* backup to oldest backup in chain, and look for the first
* full copy of destination file.
* Full copy is latest possible destination file with size equal or
* greater than zero.
*/
for (i = 1; i < parray_num(parent_chain); i++)
{
pgFile **res_file = NULL;
tmp_backup = (pgBackup *) parray_get(parent_chain, i);
/* lookup file in intermediate backup */
res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal);
tmp_file = (res_file) ? *res_file : NULL;
/*
* It should not be possible not to find destination file in intermediate
* backup, without encountering full copy first.
*/
if (!tmp_file)
{
elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s",
dest_file->rel_path, base36enc(tmp_backup->start_time));
continue;
}
/* Full copy is found and it is null sized, nothing to do here */
if (tmp_file->write_size == 0)
return;
/* Full copy is found */
if (tmp_file->write_size > 0)
break;
}
}
/* sanity */
if (!tmp_backup)
elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"",
to_fullpath);
if (!tmp_file)
elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath);
if (tmp_file->external_dir_num == 0)
// pgBackupGetPath(tmp_backup, from_root, lengthof(from_root), DATABASE_DIR);
join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR);
else
{
// get external prefix for tmp_backup
char external_prefix[MAXPGPATH];
join_path_components(external_prefix, tmp_backup->root_dir, EXTERNAL_DIR);
makeExternalDirPathByNum(from_root, external_prefix, tmp_file->external_dir_num);
}
join_path_components(from_fullpath, from_root, dest_file->rel_path);
in = fopen(from_fullpath, PG_BINARY_R);
if (in == NULL)
{
elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath,
strerror(errno));
}
restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath);
if (fio_fclose(in) != 0)
elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
strerror(errno));
}
/*
* Copy file to backup.
* We do not apply compression to these files, because

3
src/dir.c
View File

@ -1643,8 +1643,7 @@ free_dir_list(parray *list)
/* Append to string "path_prefix" int "dir_num" */
void
makeExternalDirPathByNum(char *ret_path, const char *path_prefix,
const int dir_num)
makeExternalDirPathByNum(char *ret_path, const char *path_prefix, const int dir_num)
{
sprintf(ret_path, "%s%d", path_prefix, dir_num);
}

12
src/pg_probackup.h
View File

@ -374,6 +374,8 @@ struct pgBackup
* separated by ':' */
parray *files; /* list of files belonging to this backup
* must be populated by calling backup_populate() */
char *root_dir; /* Full path for root backup directory:
backup_path/instance_name/backup_id */
};
/* Recovery target for restore and validate subcommands */
@ -837,10 +839,14 @@ extern void restore_data_file(const char *to_path,
pgFile *file, bool allow_truncate,
bool write_header,
uint32 backup_version);
extern void restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
extern void restore_data_file_new(parray *parent_chain, pgFile *dest_file,
FILE *out, const char *to_fullpath);
extern void restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
const char *from_fullpath, const char *to_fullpath, int nblocks);
extern void restore_non_data_file(FILE *in, FILE *out, pgFile *file,
const char *from_fullpath, const char *to_fullpath);
extern void restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
pgFile *dest_file, FILE *out, const char *to_fullpath);
extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
const char *from_fullpath, const char *to_fullpath);
extern bool copy_file(fio_location from_location, const char *to_root,
fio_location to_location, pgFile *file, bool missing_ok);
extern bool create_empty_file(fio_location from_location, const char *to_root,

527
src/restore.c
View File

@ -39,7 +39,6 @@ typedef struct
{
parray *dest_files;
pgBackup *dest_backup;
char *external_prefix;
parray *dest_external_dirs;
parray *parent_chain;
parray *dbOid_exclude_list;
@ -64,9 +63,9 @@ static void *restore_files(void *arg);
static void set_orphan_status(parray *backups, pgBackup *parent_backup);
static void pg12_recovery_config(pgBackup *backup, bool add_include);
static void restore_chain(pgBackup *dest_backup, parray *dest_files,
parray *parent_chain, parray *dbOid_exclude_list,
pgRestoreParams *params, const char *pgdata_path);
static void restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray *dbOid_exclude_list, pgRestoreParams *params,
const char *pgdata_path);
static void *restore_files_new(void *arg);
@ -469,21 +468,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
*/
if (params->is_restore)
{
parray *dest_external_dirs = NULL;
parray *dest_files;
char control_file[MAXPGPATH],
dest_backup_path[MAXPGPATH];
int i;
/*
* Preparations for actual restoring.
*/
pgBackupGetPath(dest_backup, control_file, lengthof(control_file),
DATABASE_FILE_LIST);
dest_files = dir_read_file_list(NULL, NULL, control_file,
FIO_BACKUP_HOST);
parray_qsort(dest_files, pgFileCompareRelPathWithExternal);
/*
* Get a list of dbOids to skip if user requested the partial restore.
* It is important that we do this after(!) validation so
@ -497,31 +481,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
*/
if (params->partial_db_list)
dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, params->partial_db_list,
params->partial_restore_type);
/*
* Restore dest_backup internal directories.
*/
pgBackupGetPath(dest_backup, dest_backup_path,
lengthof(dest_backup_path), NULL);
create_data_directories(dest_files, instance_config.pgdata, dest_backup_path, true,
FIO_DB_HOST);
/*
* Restore dest_backup external directories.
*/
if (dest_backup->external_dir_str && !params->skip_external_dirs)
{
dest_external_dirs = make_external_directory_list(
dest_backup->external_dir_str,
true);
if (parray_num(dest_external_dirs) > 0)
elog(LOG, "Restore external directories");
for (i = 0; i < parray_num(dest_external_dirs); i++)
fio_mkdir(parray_get(dest_external_dirs, i),
DIR_PERMISSION, FIO_DB_HOST);
}
params->partial_restore_type);
if (rt->lsn_string &&
parse_server_version(dest_backup->server_version) < 100000)
@ -529,9 +489,51 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
base36enc(dest_backup->start_time),
dest_backup->server_version);
/*
* Restore backups files starting from the parent backup.
*/
restore_chain(dest_backup, parent_chain, dbOid_exclude_list,
params, instance_config.pgdata);
/* Create recovery.conf with given recovery target parameters */
create_recovery_conf(target_backup_id, rt, dest_backup, params);
}
/* cleanup */
parray_walk(backups, pgBackupFree); /* free backup->files */
parray_free(backups);
parray_free(parent_chain);
elog(INFO, "%s of backup %s completed.",
action, base36enc(dest_backup->start_time));
return 0;
}
/*
* Restore backup chain.
*/
void
restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray *dbOid_exclude_list, pgRestoreParams *params,
const char *pgdata_path)
{
int i;
char control_file[MAXPGPATH];
char timestamp[100];
parray *dest_files = NULL;
parray *external_dirs = NULL;
/* arrays with meta info for multi threaded backup */
pthread_t *threads;
restore_files_arg_new *threads_args;
bool restore_isok = true;
time_t start_time, end_time;
/* Preparations for actual restoring */
time2iso(timestamp, lengthof(timestamp), dest_backup->start_time);
elog(LOG, "Restoring database from backup at %s", timestamp);
join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST);
dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
// TODO lock entire chain
// for (i = parray_num(parent_chain) - 1; i >= 0; i--)
// {
// pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
@ -545,106 +547,92 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
//
// restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list, params);
// }
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
{
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
// lock entire chain
// sanity:
// 1. check status of every backup in chain
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
if (backup->status != BACKUP_STATUS_OK &&
backup->status != BACKUP_STATUS_DONE)
{
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
if (backup->status != BACKUP_STATUS_OK &&
backup->status != BACKUP_STATUS_DONE)
{
if (params->force)
elog(WARNING, "Backup %s is not valid, restore is forced",
base36enc(backup->start_time));
else
elog(ERROR, "Backup %s cannot be restored because it is not valid",
base36enc(backup->start_time));
}
/* confirm block size compatibility */
if (backup->block_size != BLCKSZ)
elog(ERROR,
"BLCKSZ(%d) is not compatible(%d expected)",
backup->block_size, BLCKSZ);
if (backup->wal_block_size != XLOG_BLCKSZ)
elog(ERROR,
"XLOG_BLCKSZ(%d) is not compatible(%d expected)",
backup->wal_block_size, XLOG_BLCKSZ);
/* populate backup filelist */
if (backup->start_time != dest_backup->start_time)
{
pgBackupGetPath(backup, control_file, lengthof(control_file), DATABASE_FILE_LIST);
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
}
if (params->force)
elog(WARNING, "Backup %s is not valid, restore is forced",
base36enc(backup->start_time));
else
backup->files = dest_files;
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
elog(ERROR, "Backup %s cannot be restored because it is not valid",
base36enc(backup->start_time));
}
restore_chain(dest_backup, dest_files, parent_chain, dbOid_exclude_list,
params, instance_config.pgdata);
/* confirm block size compatibility */
if (backup->block_size != BLCKSZ)
elog(ERROR,
"BLCKSZ(%d) is not compatible(%d expected)",
backup->block_size, BLCKSZ);
if (dest_external_dirs != NULL)
free_dir_list(dest_external_dirs);
if (backup->wal_block_size != XLOG_BLCKSZ)
elog(ERROR,
"XLOG_BLCKSZ(%d) is not compatible(%d expected)",
backup->wal_block_size, XLOG_BLCKSZ);
parray_walk(dest_files, pgFileFree);
parray_free(dest_files);
/* populate backup filelist */
if (backup->start_time != dest_backup->start_time)
{
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST);
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
}
else
backup->files = dest_files;
/* Create recovery.conf with given recovery target parameters */
create_recovery_conf(target_backup_id, rt, dest_backup, params);
/* this sorting is important */
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
}
/* cleanup */
parray_walk(backups, pgBackupFree);
parray_free(backups);
parray_free(parent_chain);
elog(INFO, "%s of backup %s completed.",
action, base36enc(dest_backup->start_time));
return 0;
}
/*
* Restore backup chain.
*/
void
restore_chain(pgBackup *dest_backup, parray *dest_files,
parray *parent_chain, parray *dbOid_exclude_list,
pgRestoreParams *params, const char *pgdata_path)
{
char timestamp[100];
char external_prefix[MAXPGPATH];
parray *external_dirs = NULL;
int i;
/* arrays with meta info for multi threaded backup */
pthread_t *threads;
restore_files_arg_new *threads_args;
bool restore_isok = true;
time2iso(timestamp, lengthof(timestamp), dest_backup->start_time);
elog(LOG, "Restoring database from backup %s", timestamp);
if (dest_backup->external_dir_str)
external_dirs = make_external_directory_list(dest_backup->external_dir_str,
true);
/* Restore directories first */
parray_qsort(dest_files, pgFileCompareRelPathWithExternal);
/*
* Restore dest_backup internal directories.
*/
create_data_directories(dest_files, instance_config.pgdata,
dest_backup->root_dir, true, FIO_DB_HOST);
/*
* Setup file locks
* Restore dest_backup external directories.
*/
if (dest_backup->external_dir_str && !params->skip_external_dirs)
{
external_dirs = make_external_directory_list(dest_backup->external_dir_str, true);
if (!external_dirs)
elog(ERROR, "Failed to get a list of external directories");
if (parray_num(external_dirs) > 0)
elog(LOG, "Restore external directories");
for (i = 0; i < parray_num(external_dirs); i++)
fio_mkdir(parray_get(external_dirs, i),
DIR_PERMISSION, FIO_DB_HOST);
}
/*
* Setup directory structure for external directories and file locks
*/
for (i = 0; i < parray_num(dest_files); i++)
{
pgFile *file = (pgFile *) parray_get(dest_files, i);
if (!params->skip_external_dirs &&
file->external_dir_num && S_ISDIR(file->mode))
{
char *external_path;
char dirpath[MAXPGPATH];
if (parray_num(external_dirs) < file->external_dir_num - 1)
elog(ERROR, "Inconsistent external directory backup metadata");
external_path = parray_get(external_dirs, file->external_dir_num - 1);
join_path_components(dirpath, external_path, file->rel_path);
elog(VERBOSE, "Create external directory \"%s\"", dirpath);
fio_mkdir(dirpath, file->mode, FIO_DB_HOST);
}
/* setup threads */
pg_atomic_clear_flag(&file->lock);
}
@ -655,13 +643,14 @@ restore_chain(pgBackup *dest_backup, parray *dest_files,
/* Restore files into target directory */
thread_interrupted = false;
elog(INFO, "Start restoring backup files");
time(&start_time);
for (i = 0; i < num_threads; i++)
{
restore_files_arg_new *arg = &(threads_args[i]);
arg->dest_files = dest_files;
arg->dest_backup = dest_backup;
arg->external_prefix = external_prefix;
arg->dest_external_dirs = external_dirs;
arg->parent_chain = parent_chain;
arg->dbOid_exclude_list = dbOid_exclude_list;
@ -683,15 +672,59 @@ restore_chain(pgBackup *dest_backup, parray *dest_files,
if (threads_args[i].ret == 1)
restore_isok = false;
}
if (!restore_isok)
elog(ERROR, "Data files restoring failed");
time(&end_time);
if (restore_isok)
elog(INFO, "Backup files are restored, time elapsed: %.0f sec",
difftime(end_time, start_time));
else
elog(ERROR, "Backup files restoring failed, time elapsed: %.0f sec",
difftime(end_time, start_time));
elog(INFO, "Sync restored backup files to disk");
time(&start_time);
for (i = 0; i < parray_num(dest_files); i++)
{
int out;
char to_fullpath[MAXPGPATH];
pgFile *dest_file = (pgFile *) parray_get(dest_files, i);
if (S_ISDIR(dest_file->mode) ||
dest_file->external_dir_num > 0 ||
(strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) ||
(strcmp(DATABASE_MAP, dest_file->rel_path) == 0))
continue;
join_path_components(to_fullpath, pgdata_path, dest_file->rel_path);
/* open destination file */
out = fio_open(to_fullpath, O_WRONLY | PG_BINARY, FIO_DB_HOST);
if (out < 0)
elog(ERROR, "Cannot open file \"%s\": %s",
to_fullpath, strerror(errno));
/* sync file */
if (fio_flush(out) != 0 || fio_close(out) != 0)
elog(ERROR, "Cannot sync file \"%s\": %s",
to_fullpath, strerror(errno));
}
time(&end_time);
elog(INFO, "Restored backup files are synced, time elapsed: %.0f sec",
difftime(end_time, start_time));
/* cleanup */
pfree(threads);
pfree(threads_args);
if (external_dirs != NULL)
free_dir_list(external_dirs);
parray_walk(dest_files, pgFileFree);
parray_free(dest_files);
// elog(LOG, "Restore of backup %s is completed", base36enc(backup->start_time));
}
@ -701,36 +734,21 @@ restore_chain(pgBackup *dest_backup, parray *dest_files,
static void *
restore_files_new(void *arg)
{
int i, j;
int i;
char to_fullpath[MAXPGPATH];
FILE *out = NULL;
restore_files_arg_new *arguments = (restore_files_arg_new *) arg;
// for (i = parray_num(arguments->parent_chain) - 1; i >= 0; i--)
// {
// pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, i);
//
// for (j = 0; j < parray_num(backup->files); j++)
// {
// pgFile *file = (pgFile *) parray_get(backup->files, j);
//
// elog(INFO, "Backup %s;File: %s, Size: %li",
// base36enc(backup->start_time), file->name, file->write_size);
// }
// }
//
// elog(ERROR, "HELLO");
for (i = 0; i < parray_num(arguments->dest_files); i++)
{
pgFile *file = (pgFile *) parray_get(arguments->dest_files, i);
pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i);
/* Directories were created before */
if (S_ISDIR(file->mode))
if (S_ISDIR(dest_file->mode))
continue;
if (!pg_atomic_test_set_flag(&file->lock))
if (!pg_atomic_test_set_flag(&dest_file->lock))
continue;
/* check for interrupt */
@ -740,53 +758,57 @@ restore_files_new(void *arg)
if (progress)
elog(INFO, "Progress: (%d/%lu). Process file %s ",
i + 1, (unsigned long) parray_num(arguments->dest_files),
file->rel_path);
dest_file->rel_path);
/* Only files from pgdata can be skipped by partial restore */
if (arguments->dbOid_exclude_list && file->external_dir_num == 0)
if (arguments->dbOid_exclude_list && dest_file->external_dir_num == 0)
{
/* Check if the file belongs to the database we exclude */
if (parray_bsearch(arguments->dbOid_exclude_list,
&file->dbOid, pgCompareOid))
&dest_file->dbOid, pgCompareOid))
{
/*
* We cannot simply skip the file, because it may lead to
* failure during WAL redo; hence, create empty file.
* failure during WAL redo; hence, create empty file.
*/
create_empty_file(FIO_BACKUP_HOST,
arguments->to_root, FIO_DB_HOST, file);
arguments->to_root, FIO_DB_HOST, dest_file);
elog(VERBOSE, "Exclude file due to partial restore: \"%s\"",
file->rel_path);
elog(INFO, "Skip file due to partial restore: \"%s\"",
dest_file->rel_path);
continue;
}
}
/* Do not restore tablespace_map file */
if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path))
if ((dest_file->external_dir_num == 0) &&
strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0)
{
elog(VERBOSE, "Skip tablespace_map");
continue;
}
/* Do not restore database_map file */
if ((file->external_dir_num == 0) &&
strcmp(DATABASE_MAP, file->rel_path) == 0)
if ((dest_file->external_dir_num == 0) &&
strcmp(DATABASE_MAP, dest_file->rel_path) == 0)
{
elog(VERBOSE, "Skip database_map");
continue;
}
/* Do no restore external directory file if a user doesn't want */
if (arguments->skip_external_dirs && file->external_dir_num > 0)
if (arguments->skip_external_dirs && dest_file->external_dir_num > 0)
continue;
//set max_blknum based on file->n_blocks
/* set fullpath of destination file */
if (file->external_dir_num == 0)
join_path_components(to_fullpath, arguments->to_root, file->rel_path);
//else
// TODO
if (dest_file->external_dir_num == 0)
join_path_components(to_fullpath, arguments->to_root, dest_file->rel_path);
else
{
char *external_path = parray_get(arguments->dest_external_dirs,
dest_file->external_dir_num - 1);
join_path_components(to_fullpath, external_path, dest_file->rel_path);
}
/* open destination file */
out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST);
@ -797,158 +819,32 @@ restore_files_new(void *arg)
to_fullpath, strerror(errno_tmp));
}
if (!file->is_datafile || file->is_cfs)
if (!dest_file->is_datafile || dest_file->is_cfs)
elog(VERBOSE, "Restoring non-data file: \"%s\"", to_fullpath);
else
elog(VERBOSE, "Restoring data file: \"%s\"", to_fullpath);
// if dest file is 0 sized, then just close it and go for the next
if (file->write_size == 0)
// If destination file is 0 sized, then just close it and go for the next
if (dest_file->write_size == 0)
goto done;
// TODO
// optimize copying of non-data files:
// lookup latest backup with file that has not BYTES_INVALID size
// and copy only it.
/* Restore destination file */
if (dest_file->is_datafile && !dest_file->is_cfs)
/* Destination file is data file */
restore_data_file_new(arguments->parent_chain, dest_file, out, to_fullpath);
else
/* Destination file is non-data file */
restore_non_data_file(arguments->parent_chain, arguments->dest_backup,
dest_file, out, to_fullpath);
if (!file->is_datafile || file->is_cfs)
{
char from_root[MAXPGPATH];
char from_fullpath[MAXPGPATH];
FILE *in = NULL;
pgFile *tmp_file = NULL;
if (file->write_size > 0)
{
tmp_file = file;
pgBackupGetPath(arguments->dest_backup, from_root, lengthof(from_root), DATABASE_DIR);
join_path_components(from_fullpath, from_root, file->rel_path);
}
else
{
for (j = 0; j < parray_num(arguments->parent_chain); j++)
{
pgFile **res_file = NULL;
pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, j);
/* lookup file in intermediate backup */
res_file = parray_bsearch(backup->files, file, pgFileCompareRelPathWithExternal);
tmp_file = (res_file) ? *res_file : NULL;
if (!tmp_file)
continue;
if (tmp_file->write_size == 0)
goto done;
if (tmp_file->write_size > 0)
{
pgBackupGetPath(backup, from_root, lengthof(from_root), DATABASE_DIR);
join_path_components(from_fullpath, from_root, file->rel_path);
break;
}
}
}
if (!tmp_file)
elog(ERROR, "Something went wrong");
in = fopen(from_fullpath, PG_BINARY_R);
if (in == NULL)
{
elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath,
strerror(errno));
}
if (strcmp(file->name, "pg_control") == 0)
copy_pgcontrol_file(from_root, FIO_BACKUP_HOST,
instance_config.pgdata, FIO_DB_HOST,
file);
else
restore_non_data_file(in, out, tmp_file, from_fullpath, to_fullpath);
if (fio_fclose(in) != 0)
elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
strerror(errno));
goto done;
}
for (j = parray_num(arguments->parent_chain) - 1; j >= 0; j--)
{
char from_root[MAXPGPATH];
char from_fullpath[MAXPGPATH];
FILE *in = NULL;
pgFile **res_file = NULL;
pgFile *tmp_file = NULL;
pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, j);
/* lookup file in intermediate backup */
res_file = parray_bsearch(backup->files, file, pgFileCompareRelPathWithExternal);
tmp_file = (res_file) ? *res_file : NULL;
/* destination file is not exists in this intermediate backup */
if (tmp_file == NULL)
continue;
/* check for interrupt */
if (interrupted || thread_interrupted)
elog(ERROR, "Interrupted during restore");
/*
* For incremental backups skip files which haven't changed
* since previous backup and thus were not backed up.
*/
if (tmp_file->write_size == BYTES_INVALID)
{
elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", from_fullpath);
continue;
}
/*
* At this point we are sure, that something is going to be copied
* Open source file.
*/
/* TODO: special handling for files from external directories */
pgBackupGetPath(backup, from_root, lengthof(from_root), DATABASE_DIR);
join_path_components(from_fullpath, from_root, file->rel_path);
in = fopen(from_fullpath, PG_BINARY_R);
if (in == NULL)
{
elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath,
strerror(errno));
}
/*
* restore the file.
* We treat datafiles separately, cause they were backed up block by
* block and have BackupPageHeader meta information, so we cannot just
* copy the file from backup.
*/
// if (file->is_datafile && file->is_cfs)
restore_data_file_new(in, out, tmp_file,
parse_program_version(backup->program_version),
from_fullpath, to_fullpath, file->n_blocks);
// else if (strcmp(file->name, "pg_control") == 0)
// copy_pgcontrol_file(from_root, FIO_BACKUP_HOST,
// instance_config.pgdata, FIO_DB_HOST,
// file);
// else
// restore_non_data_file(in, out, tmp_file, from_fullpath, to_fullpath);
if (fio_fclose(in) != 0)
elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
strerror(errno));
}
/*
* Destination file is data file.
* Iterate over incremental chain and lookup given destination file.
* Apply changed blocks to destination file from every backup in parent chain.
*/
done:
// chmod
// fsync
// close
/* truncate file up to n_blocks. NOTE: no need, we just should not write
@ -957,7 +853,7 @@ restore_files_new(void *arg)
*/
/* update file permission */
if (fio_chmod(to_fullpath, file->mode, FIO_DB_HOST) == -1)
if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1)
{
int errno_tmp = errno;
fio_fclose(out);
@ -965,13 +861,6 @@ restore_files_new(void *arg)
strerror(errno_tmp));
}
/* flush file */
if (fio_fflush(out) != 0)
elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath,
strerror(errno));
/* fsync file */
/* close file */
if (fio_fclose(out) != 0)
elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath,
@ -1903,7 +1792,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
elog(ERROR, "Backup %s doesn't contain a database_map, partial restore is impossible.",
base36enc(backup->start_time));
pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR);
join_path_components(path, backup->root_dir, DATABASE_DIR);
join_path_components(database_map_path, path, DATABASE_MAP);
/* check database_map CRC */

9
src/utils/file.c
View File

@ -447,17 +447,12 @@ int fio_fprintf(FILE* f, char const* format, ...)
return rc;
}
/* Flush stream data (does nothing for remote file) */
/* Flush stream data (does nothing for remote file) */
int fio_fflush(FILE* f)
{
int rc = 0;
if (!fio_is_remote_file(f))
{
rc = fflush(f);
if (rc == 0) {
rc = fsync(fileno(f));
}
}
return rc;
}
@ -560,7 +555,7 @@ int fio_pread(FILE* f, void* buf, off_t offs)
int fio_fseek(FILE* f, off_t offs)
{
return fio_is_remote_file(f)
? fio_seek(fio_fileno(f), offs)
? fio_seek(fio_fileno(f), offs)
: fseek(f, offs, SEEK_SET);
}