diff --git a/src/catalog.c b/src/catalog.c index 1fe8e4b2..d6bc2dd4 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -444,6 +444,9 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) base36enc(backup->start_time), backup_conf_path); } + backup->root_dir = pgut_strdup(data_path); + + /* TODO: save encoded backup id */ backup->backup_id = backup->start_time; if (requested_backup_id != INVALID_BACKUP_ID && requested_backup_id != backup->start_time) @@ -2005,6 +2008,7 @@ pgBackupInit(pgBackup *backup) backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; backup->external_dir_str = NULL; + backup->root_dir = NULL; } /* free pgBackup object */ @@ -2015,6 +2019,7 @@ pgBackupFree(void *backup) pfree(b->primary_conninfo); pfree(b->external_dir_str); + pfree(b->root_dir); pfree(backup); } diff --git a/src/data.c b/src/data.c index 1d0fada2..572ee750 100644 --- a/src/data.c +++ b/src/data.c @@ -962,23 +962,88 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, } /* - * Restore files in the from_root directory to the to_root directory with - * same relative path. - * - * If write_header is true then we add header to each restored block, currently - * it is used for MERGE command. - * - * to_fullpath and from_fullpath are provided strictly for ERROR reporting + * Iterate over parent backup chain and lookup given destination file in + * filelist of every chain member starting with FULL backup. + * Apply changed blocks to destination file from every backup in parent chain. */ void -restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, +restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath) +{ + int i; + + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + char from_root[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + FILE *in = NULL; + + pgFile **res_file = NULL; + pgFile *tmp_file = NULL; + + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during restore"); + + /* lookup file in intermediate backup */ + res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); + tmp_file = (res_file) ? *res_file : NULL; + + /* Destination file is not exists yet at this moment */ + if (tmp_file == NULL) + continue; + + /* + * Skip file if it haven't changed since previous backup + * and thus was not backed up. + */ + if (tmp_file->write_size == BYTES_INVALID) + { +// elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", tmp_file->rel_path); + continue; + } + + /* + * At this point we are sure, that something is going to be copied + * Open source file. + */ + join_path_components(from_root, backup->root_dir, DATABASE_DIR); + join_path_components(from_fullpath, from_root, tmp_file->rel_path); + + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + elog(INFO, "Cannot open backup file \"%s\": %s", from_fullpath, + strerror(errno)); + Assert(0); + } + + /* + * restore the file. + * Datafiles are backed up block by block and every block + * have BackupPageHeader with meta information, so we cannot just + * copy the file from backup. + */ + restore_data_file_internal(in, out, tmp_file, + parse_program_version(backup->program_version), + from_fullpath, to_fullpath, dest_file->n_blocks); + + if (fio_fclose(in) != 0) + elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, + strerror(errno)); + } +} + +void +restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks) { BackupPageHeader header; BlockNumber blknum = 0; size_t write_len = 0; - while (true) + for (;;) { off_t write_pos; size_t read_len; @@ -1016,9 +1081,44 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, blknum = header.block; + /* + * Backupward compatibility kludge: in the good old days + * n_blocks attribute was available only in DELTA backups. + * File truncate in PAGE and PTRACK happened on the fly when + * special value PageIsTruncated is encountered. + * It is inefficient. + * + * Nowadays every backup type has n_blocks, so instead + * writing and then truncating redundant data, writing + * is not happening in the first place. + * TODO: remove in 3.0.0 + */ + if (header.compressed_size == PageIsTruncated) + { + /* + * Block header contains information that this block was truncated. + * We need to truncate file to this length. + */ + + elog(VERBOSE, "Truncate file \"%s\" to block %u", to_fullpath, header.block); + + /* To correctly truncate file, we must first flush STDIO buffers */ + if (fio_fflush(out) != 0) + elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno)); + + /* Set position to the start of file */ + if (fio_fseek(out, 0) < 0) + elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno)); + + if (fio_ftruncate(out, header.block * BLCKSZ) != 0) + elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno)); + + break; + } + /* no point in writing redundant data */ if (nblocks > 0 && blknum >= nblocks) - return; + break; if (header.compressed_size > BLCKSZ) elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); @@ -1061,7 +1161,6 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, /* * Seek and write the restored page. - * TODO: invent fio_pwrite(). */ if (fio_fseek(out, write_pos) < 0) elog(ERROR, "Cannot seek block %u of \"%s\": %s", @@ -1096,7 +1195,7 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, * it is either small control file or already compressed cfs file. */ void -restore_non_data_file(FILE *in, FILE *out, pgFile *file, +restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, const char *from_fullpath, const char *to_fullpath) { size_t read_len = 0; @@ -1148,6 +1247,100 @@ restore_non_data_file(FILE *in, FILE *out, pgFile *file, elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); } +void +restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, + pgFile *dest_file, FILE *out, const char *to_fullpath) +{ + int i; + char from_root[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + FILE *in = NULL; + + pgFile *tmp_file = NULL; + pgBackup *tmp_backup = NULL; + + /* Check if full copy of destination file is available in destination backup */ + if (dest_file->write_size > 0) + { + tmp_file = dest_file; + tmp_backup = dest_backup; + } + else + { + /* + * Iterate over parent chain starting from direct parent of destination + * backup to oldest backup in chain, and look for the first + * full copy of destination file. + * Full copy is latest possible destination file with size equal or + * greater than zero. + */ + for (i = 1; i < parray_num(parent_chain); i++) + { + pgFile **res_file = NULL; + + tmp_backup = (pgBackup *) parray_get(parent_chain, i); + + /* lookup file in intermediate backup */ + res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal); + tmp_file = (res_file) ? *res_file : NULL; + + /* + * It should not be possible not to find destination file in intermediate + * backup, without encountering full copy first. + */ + if (!tmp_file) + { + elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", + dest_file->rel_path, base36enc(tmp_backup->start_time)); + continue; + } + + /* Full copy is found and it is null sized, nothing to do here */ + if (tmp_file->write_size == 0) + return; + + /* Full copy is found */ + if (tmp_file->write_size > 0) + break; + } + } + + /* sanity */ + if (!tmp_backup) + elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"", + to_fullpath); + + if (!tmp_file) + elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); + + if (tmp_file->external_dir_num == 0) +// pgBackupGetPath(tmp_backup, from_root, lengthof(from_root), DATABASE_DIR); + join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR); + else + { + // get external prefix for tmp_backup + char external_prefix[MAXPGPATH]; + + join_path_components(external_prefix, tmp_backup->root_dir, EXTERNAL_DIR); + makeExternalDirPathByNum(from_root, external_prefix, tmp_file->external_dir_num); + } + + join_path_components(from_fullpath, from_root, dest_file->rel_path); + + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, + strerror(errno)); + } + + restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); + + if (fio_fclose(in) != 0) + elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, + strerror(errno)); +} + /* * Copy file to backup. * We do not apply compression to these files, because diff --git a/src/dir.c b/src/dir.c index 754e6bde..496518ad 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1643,8 +1643,7 @@ free_dir_list(parray *list) /* Append to string "path_prefix" int "dir_num" */ void -makeExternalDirPathByNum(char *ret_path, const char *path_prefix, - const int dir_num) +makeExternalDirPathByNum(char *ret_path, const char *path_prefix, const int dir_num) { sprintf(ret_path, "%s%d", path_prefix, dir_num); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d5d5f2b4..d21d8540 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -374,6 +374,8 @@ struct pgBackup * separated by ':' */ parray *files; /* list of files belonging to this backup * must be populated by calling backup_populate() */ + char *root_dir; /* Full path for root backup directory: + backup_path/instance_name/backup_id */ }; /* Recovery target for restore and validate subcommands */ @@ -837,10 +839,14 @@ extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header, uint32 backup_version); -extern void restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, +extern void restore_data_file_new(parray *parent_chain, pgFile *dest_file, + FILE *out, const char *to_fullpath); +extern void restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks); -extern void restore_non_data_file(FILE *in, FILE *out, pgFile *file, - const char *from_fullpath, const char *to_fullpath); +extern void restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, + pgFile *dest_file, FILE *out, const char *to_fullpath); +extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, + const char *from_fullpath, const char *to_fullpath); extern bool copy_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file, bool missing_ok); extern bool create_empty_file(fio_location from_location, const char *to_root, diff --git a/src/restore.c b/src/restore.c index 13b6184d..16f3186c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -39,7 +39,6 @@ typedef struct { parray *dest_files; pgBackup *dest_backup; - char *external_prefix; parray *dest_external_dirs; parray *parent_chain; parray *dbOid_exclude_list; @@ -64,9 +63,9 @@ static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); static void pg12_recovery_config(pgBackup *backup, bool add_include); -static void restore_chain(pgBackup *dest_backup, parray *dest_files, - parray *parent_chain, parray *dbOid_exclude_list, - pgRestoreParams *params, const char *pgdata_path); +static void restore_chain(pgBackup *dest_backup, parray *parent_chain, + parray *dbOid_exclude_list, pgRestoreParams *params, + const char *pgdata_path); static void *restore_files_new(void *arg); @@ -469,21 +468,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->is_restore) { - parray *dest_external_dirs = NULL; - parray *dest_files; - char control_file[MAXPGPATH], - dest_backup_path[MAXPGPATH]; - int i; - - /* - * Preparations for actual restoring. - */ - pgBackupGetPath(dest_backup, control_file, lengthof(control_file), - DATABASE_FILE_LIST); - dest_files = dir_read_file_list(NULL, NULL, control_file, - FIO_BACKUP_HOST); - parray_qsort(dest_files, pgFileCompareRelPathWithExternal); - /* * Get a list of dbOids to skip if user requested the partial restore. * It is important that we do this after(!) validation so @@ -497,31 +481,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->partial_db_list) dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, params->partial_db_list, - params->partial_restore_type); - - /* - * Restore dest_backup internal directories. - */ - pgBackupGetPath(dest_backup, dest_backup_path, - lengthof(dest_backup_path), NULL); - create_data_directories(dest_files, instance_config.pgdata, dest_backup_path, true, - FIO_DB_HOST); - - /* - * Restore dest_backup external directories. - */ - if (dest_backup->external_dir_str && !params->skip_external_dirs) - { - dest_external_dirs = make_external_directory_list( - dest_backup->external_dir_str, - true); - if (parray_num(dest_external_dirs) > 0) - elog(LOG, "Restore external directories"); - - for (i = 0; i < parray_num(dest_external_dirs); i++) - fio_mkdir(parray_get(dest_external_dirs, i), - DIR_PERMISSION, FIO_DB_HOST); - } + params->partial_restore_type); if (rt->lsn_string && parse_server_version(dest_backup->server_version) < 100000) @@ -529,9 +489,51 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time), dest_backup->server_version); - /* - * Restore backups files starting from the parent backup. - */ + restore_chain(dest_backup, parent_chain, dbOid_exclude_list, + params, instance_config.pgdata); + + /* Create recovery.conf with given recovery target parameters */ + create_recovery_conf(target_backup_id, rt, dest_backup, params); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); /* free backup->files */ + parray_free(backups); + parray_free(parent_chain); + + elog(INFO, "%s of backup %s completed.", + action, base36enc(dest_backup->start_time)); + return 0; +} + +/* + * Restore backup chain. + */ +void +restore_chain(pgBackup *dest_backup, parray *parent_chain, + parray *dbOid_exclude_list, pgRestoreParams *params, + const char *pgdata_path) +{ + int i; + char control_file[MAXPGPATH]; + char timestamp[100]; + parray *dest_files = NULL; + parray *external_dirs = NULL; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + restore_files_arg_new *threads_args; + bool restore_isok = true; + + time_t start_time, end_time; + + /* Preparations for actual restoring */ + time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); + elog(LOG, "Restoring database from backup at %s", timestamp); + + join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST); + dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); + + // TODO lock entire chain // for (i = parray_num(parent_chain) - 1; i >= 0; i--) // { // pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); @@ -545,106 +547,92 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, // // restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list, params); // } + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - // lock entire chain - - // sanity: - // 1. check status of every backup in chain - for (i = parray_num(parent_chain) - 1; i >= 0; i--) + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE) - { - if (params->force) - elog(WARNING, "Backup %s is not valid, restore is forced", - base36enc(backup->start_time)); - else - elog(ERROR, "Backup %s cannot be restored because it is not valid", - base36enc(backup->start_time)); - } - - /* confirm block size compatibility */ - if (backup->block_size != BLCKSZ) - elog(ERROR, - "BLCKSZ(%d) is not compatible(%d expected)", - backup->block_size, BLCKSZ); - - if (backup->wal_block_size != XLOG_BLCKSZ) - elog(ERROR, - "XLOG_BLCKSZ(%d) is not compatible(%d expected)", - backup->wal_block_size, XLOG_BLCKSZ); - - /* populate backup filelist */ - if (backup->start_time != dest_backup->start_time) - { - pgBackupGetPath(backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); - } + if (params->force) + elog(WARNING, "Backup %s is not valid, restore is forced", + base36enc(backup->start_time)); else - backup->files = dest_files; - - parray_qsort(backup->files, pgFileCompareRelPathWithExternal); + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); } - restore_chain(dest_backup, dest_files, parent_chain, dbOid_exclude_list, - params, instance_config.pgdata); + /* confirm block size compatibility */ + if (backup->block_size != BLCKSZ) + elog(ERROR, + "BLCKSZ(%d) is not compatible(%d expected)", + backup->block_size, BLCKSZ); - if (dest_external_dirs != NULL) - free_dir_list(dest_external_dirs); + if (backup->wal_block_size != XLOG_BLCKSZ) + elog(ERROR, + "XLOG_BLCKSZ(%d) is not compatible(%d expected)", + backup->wal_block_size, XLOG_BLCKSZ); - parray_walk(dest_files, pgFileFree); - parray_free(dest_files); + /* populate backup filelist */ + if (backup->start_time != dest_backup->start_time) + { + join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST); + backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); + } + else + backup->files = dest_files; - /* Create recovery.conf with given recovery target parameters */ - create_recovery_conf(target_backup_id, rt, dest_backup, params); + /* this sorting is important */ + parray_qsort(backup->files, pgFileCompareRelPathWithExternal); } - /* cleanup */ - parray_walk(backups, pgBackupFree); - parray_free(backups); - parray_free(parent_chain); - - elog(INFO, "%s of backup %s completed.", - action, base36enc(dest_backup->start_time)); - return 0; -} - -/* - * Restore backup chain. - */ -void -restore_chain(pgBackup *dest_backup, parray *dest_files, - parray *parent_chain, parray *dbOid_exclude_list, - pgRestoreParams *params, const char *pgdata_path) -{ - char timestamp[100]; - char external_prefix[MAXPGPATH]; - parray *external_dirs = NULL; - int i; - /* arrays with meta info for multi threaded backup */ - pthread_t *threads; - restore_files_arg_new *threads_args; - bool restore_isok = true; - - time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); - elog(LOG, "Restoring database from backup %s", timestamp); - - if (dest_backup->external_dir_str) - external_dirs = make_external_directory_list(dest_backup->external_dir_str, - true); - - /* Restore directories first */ - parray_qsort(dest_files, pgFileCompareRelPathWithExternal); + /* + * Restore dest_backup internal directories. + */ + create_data_directories(dest_files, instance_config.pgdata, + dest_backup->root_dir, true, FIO_DB_HOST); /* - * Setup file locks + * Restore dest_backup external directories. + */ + if (dest_backup->external_dir_str && !params->skip_external_dirs) + { + external_dirs = make_external_directory_list(dest_backup->external_dir_str, true); + + if (!external_dirs) + elog(ERROR, "Failed to get a list of external directories"); + + if (parray_num(external_dirs) > 0) + elog(LOG, "Restore external directories"); + + for (i = 0; i < parray_num(external_dirs); i++) + fio_mkdir(parray_get(external_dirs, i), + DIR_PERMISSION, FIO_DB_HOST); + } + + /* + * Setup directory structure for external directories and file locks */ for (i = 0; i < parray_num(dest_files); i++) { pgFile *file = (pgFile *) parray_get(dest_files, i); + if (!params->skip_external_dirs && + file->external_dir_num && S_ISDIR(file->mode)) + { + char *external_path; + char dirpath[MAXPGPATH]; + + if (parray_num(external_dirs) < file->external_dir_num - 1) + elog(ERROR, "Inconsistent external directory backup metadata"); + + external_path = parray_get(external_dirs, file->external_dir_num - 1); + join_path_components(dirpath, external_path, file->rel_path); + + elog(VERBOSE, "Create external directory \"%s\"", dirpath); + fio_mkdir(dirpath, file->mode, FIO_DB_HOST); + } + /* setup threads */ pg_atomic_clear_flag(&file->lock); } @@ -655,13 +643,14 @@ restore_chain(pgBackup *dest_backup, parray *dest_files, /* Restore files into target directory */ thread_interrupted = false; + elog(INFO, "Start restoring backup files"); + time(&start_time); for (i = 0; i < num_threads; i++) { restore_files_arg_new *arg = &(threads_args[i]); arg->dest_files = dest_files; arg->dest_backup = dest_backup; - arg->external_prefix = external_prefix; arg->dest_external_dirs = external_dirs; arg->parent_chain = parent_chain; arg->dbOid_exclude_list = dbOid_exclude_list; @@ -683,15 +672,59 @@ restore_chain(pgBackup *dest_backup, parray *dest_files, if (threads_args[i].ret == 1) restore_isok = false; } - if (!restore_isok) - elog(ERROR, "Data files restoring failed"); + time(&end_time); + if (restore_isok) + elog(INFO, "Backup files are restored, time elapsed: %.0f sec", + difftime(end_time, start_time)); + else + elog(ERROR, "Backup files restoring failed, time elapsed: %.0f sec", + difftime(end_time, start_time)); + + + elog(INFO, "Sync restored backup files to disk"); + time(&start_time); + + for (i = 0; i < parray_num(dest_files); i++) + { + int out; + char to_fullpath[MAXPGPATH]; + pgFile *dest_file = (pgFile *) parray_get(dest_files, i); + + if (S_ISDIR(dest_file->mode) || + dest_file->external_dir_num > 0 || + (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) || + (strcmp(DATABASE_MAP, dest_file->rel_path) == 0)) + continue; + + join_path_components(to_fullpath, pgdata_path, dest_file->rel_path); + + /* open destination file */ + out = fio_open(to_fullpath, O_WRONLY | PG_BINARY, FIO_DB_HOST); + if (out < 0) + elog(ERROR, "Cannot open file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* sync file */ + if (fio_flush(out) != 0 || fio_close(out) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", + to_fullpath, strerror(errno)); + } + + time(&end_time); + elog(INFO, "Restored backup files are synced, time elapsed: %.0f sec", + difftime(end_time, start_time)); + + /* cleanup */ pfree(threads); pfree(threads_args); if (external_dirs != NULL) free_dir_list(external_dirs); + parray_walk(dest_files, pgFileFree); + parray_free(dest_files); + // elog(LOG, "Restore of backup %s is completed", base36enc(backup->start_time)); } @@ -701,36 +734,21 @@ restore_chain(pgBackup *dest_backup, parray *dest_files, static void * restore_files_new(void *arg) { - int i, j; + int i; char to_fullpath[MAXPGPATH]; FILE *out = NULL; restore_files_arg_new *arguments = (restore_files_arg_new *) arg; -// for (i = parray_num(arguments->parent_chain) - 1; i >= 0; i--) -// { -// pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, i); -// -// for (j = 0; j < parray_num(backup->files); j++) -// { -// pgFile *file = (pgFile *) parray_get(backup->files, j); -// -// elog(INFO, "Backup %s;File: %s, Size: %li", -// base36enc(backup->start_time), file->name, file->write_size); -// } -// } -// -// elog(ERROR, "HELLO"); - for (i = 0; i < parray_num(arguments->dest_files); i++) { - pgFile *file = (pgFile *) parray_get(arguments->dest_files, i); + pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); /* Directories were created before */ - if (S_ISDIR(file->mode)) + if (S_ISDIR(dest_file->mode)) continue; - if (!pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&dest_file->lock)) continue; /* check for interrupt */ @@ -740,53 +758,57 @@ restore_files_new(void *arg) if (progress) elog(INFO, "Progress: (%d/%lu). Process file %s ", i + 1, (unsigned long) parray_num(arguments->dest_files), - file->rel_path); + dest_file->rel_path); /* Only files from pgdata can be skipped by partial restore */ - if (arguments->dbOid_exclude_list && file->external_dir_num == 0) + if (arguments->dbOid_exclude_list && dest_file->external_dir_num == 0) { /* Check if the file belongs to the database we exclude */ if (parray_bsearch(arguments->dbOid_exclude_list, - &file->dbOid, pgCompareOid)) + &dest_file->dbOid, pgCompareOid)) { /* * We cannot simply skip the file, because it may lead to - * failure during WAL redo; hence, create empty file. + * failure during WAL redo; hence, create empty file. */ create_empty_file(FIO_BACKUP_HOST, - arguments->to_root, FIO_DB_HOST, file); + arguments->to_root, FIO_DB_HOST, dest_file); - elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", - file->rel_path); + elog(INFO, "Skip file due to partial restore: \"%s\"", + dest_file->rel_path); continue; } } /* Do not restore tablespace_map file */ - if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path)) + if ((dest_file->external_dir_num == 0) && + strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) { elog(VERBOSE, "Skip tablespace_map"); continue; } /* Do not restore database_map file */ - if ((file->external_dir_num == 0) && - strcmp(DATABASE_MAP, file->rel_path) == 0) + if ((dest_file->external_dir_num == 0) && + strcmp(DATABASE_MAP, dest_file->rel_path) == 0) { elog(VERBOSE, "Skip database_map"); continue; } /* Do no restore external directory file if a user doesn't want */ - if (arguments->skip_external_dirs && file->external_dir_num > 0) + if (arguments->skip_external_dirs && dest_file->external_dir_num > 0) continue; - //set max_blknum based on file->n_blocks /* set fullpath of destination file */ - if (file->external_dir_num == 0) - join_path_components(to_fullpath, arguments->to_root, file->rel_path); - //else - // TODO + if (dest_file->external_dir_num == 0) + join_path_components(to_fullpath, arguments->to_root, dest_file->rel_path); + else + { + char *external_path = parray_get(arguments->dest_external_dirs, + dest_file->external_dir_num - 1); + join_path_components(to_fullpath, external_path, dest_file->rel_path); + } /* open destination file */ out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); @@ -797,158 +819,32 @@ restore_files_new(void *arg) to_fullpath, strerror(errno_tmp)); } - if (!file->is_datafile || file->is_cfs) + if (!dest_file->is_datafile || dest_file->is_cfs) elog(VERBOSE, "Restoring non-data file: \"%s\"", to_fullpath); else elog(VERBOSE, "Restoring data file: \"%s\"", to_fullpath); - // if dest file is 0 sized, then just close it and go for the next - if (file->write_size == 0) + // If destination file is 0 sized, then just close it and go for the next + if (dest_file->write_size == 0) goto done; - // TODO - // optimize copying of non-data files: - // lookup latest backup with file that has not BYTES_INVALID size - // and copy only it. + /* Restore destination file */ + if (dest_file->is_datafile && !dest_file->is_cfs) + /* Destination file is data file */ + restore_data_file_new(arguments->parent_chain, dest_file, out, to_fullpath); + else + /* Destination file is non-data file */ + restore_non_data_file(arguments->parent_chain, arguments->dest_backup, + dest_file, out, to_fullpath); - if (!file->is_datafile || file->is_cfs) - { - char from_root[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - FILE *in = NULL; - - pgFile *tmp_file = NULL; - - if (file->write_size > 0) - { - tmp_file = file; - pgBackupGetPath(arguments->dest_backup, from_root, lengthof(from_root), DATABASE_DIR); - join_path_components(from_fullpath, from_root, file->rel_path); - } - else - { - for (j = 0; j < parray_num(arguments->parent_chain); j++) - { - pgFile **res_file = NULL; - pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, j); - - /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, file, pgFileCompareRelPathWithExternal); - tmp_file = (res_file) ? *res_file : NULL; - - if (!tmp_file) - continue; - - if (tmp_file->write_size == 0) - goto done; - - if (tmp_file->write_size > 0) - { - pgBackupGetPath(backup, from_root, lengthof(from_root), DATABASE_DIR); - join_path_components(from_fullpath, from_root, file->rel_path); - break; - } - } - } - - if (!tmp_file) - elog(ERROR, "Something went wrong"); - - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, - strerror(errno)); - } - - if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, - instance_config.pgdata, FIO_DB_HOST, - file); - else - restore_non_data_file(in, out, tmp_file, from_fullpath, to_fullpath); - - if (fio_fclose(in) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, - strerror(errno)); - - goto done; - } - - for (j = parray_num(arguments->parent_chain) - 1; j >= 0; j--) - { - char from_root[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - FILE *in = NULL; - - pgFile **res_file = NULL; - pgFile *tmp_file = NULL; - - pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, j); - - /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, file, pgFileCompareRelPathWithExternal); - tmp_file = (res_file) ? *res_file : NULL; - - /* destination file is not exists in this intermediate backup */ - if (tmp_file == NULL) - continue; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during restore"); - - /* - * For incremental backups skip files which haven't changed - * since previous backup and thus were not backed up. - */ - if (tmp_file->write_size == BYTES_INVALID) - { - elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", from_fullpath); - continue; - } - - /* - * At this point we are sure, that something is going to be copied - * Open source file. - */ - - /* TODO: special handling for files from external directories */ - pgBackupGetPath(backup, from_root, lengthof(from_root), DATABASE_DIR); - join_path_components(from_fullpath, from_root, file->rel_path); - - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, - strerror(errno)); - } - - /* - * restore the file. - * We treat datafiles separately, cause they were backed up block by - * block and have BackupPageHeader meta information, so we cannot just - * copy the file from backup. - */ -// if (file->is_datafile && file->is_cfs) - restore_data_file_new(in, out, tmp_file, - parse_program_version(backup->program_version), - from_fullpath, to_fullpath, file->n_blocks); -// else if (strcmp(file->name, "pg_control") == 0) -// copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, -// instance_config.pgdata, FIO_DB_HOST, -// file); -// else -// restore_non_data_file(in, out, tmp_file, from_fullpath, to_fullpath); - - if (fio_fclose(in) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, - strerror(errno)); - } + /* + * Destination file is data file. + * Iterate over incremental chain and lookup given destination file. + * Apply changed blocks to destination file from every backup in parent chain. + */ done: // chmod - // fsync // close /* truncate file up to n_blocks. NOTE: no need, we just should not write @@ -957,7 +853,7 @@ restore_files_new(void *arg) */ /* update file permission */ - if (fio_chmod(to_fullpath, file->mode, FIO_DB_HOST) == -1) + if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) { int errno_tmp = errno; fio_fclose(out); @@ -965,13 +861,6 @@ restore_files_new(void *arg) strerror(errno_tmp)); } - /* flush file */ - if (fio_fflush(out) != 0) - elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, - strerror(errno)); - - /* fsync file */ - /* close file */ if (fio_fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, @@ -1903,7 +1792,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, elog(ERROR, "Backup %s doesn't contain a database_map, partial restore is impossible.", base36enc(backup->start_time)); - pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); /* check database_map CRC */ diff --git a/src/utils/file.c b/src/utils/file.c index f7c3eab7..bae08ef2 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -447,17 +447,12 @@ int fio_fprintf(FILE* f, char const* format, ...) return rc; } -/* Flush stream data (does nothing for remote file) */ +/* Flush stream data (does nothing for remote file) */ int fio_fflush(FILE* f) { int rc = 0; if (!fio_is_remote_file(f)) - { rc = fflush(f); - if (rc == 0) { - rc = fsync(fileno(f)); - } - } return rc; } @@ -560,7 +555,7 @@ int fio_pread(FILE* f, void* buf, off_t offs) int fio_fseek(FILE* f, off_t offs) { return fio_is_remote_file(f) - ? fio_seek(fio_fileno(f), offs) + ? fio_seek(fio_fileno(f), offs) : fseek(f, offs, SEEK_SET); }