diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 32260d93..81175873 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -68,6 +68,7 @@ static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; bool progress = false; +bool no_sync = false; #if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; #endif @@ -155,13 +156,14 @@ static void opt_datname_include_list(ConfigOption *opt, const char *arg); static ConfigOption cmd_options[] = { /* directory options */ - { 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT }, + { 'b', 120, "help", &help_opt, SOURCE_CMD_STRICT }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT }, /* common options */ { 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT }, - { 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT }, - { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, + { 'b', 121, "stream", &stream_wal, SOURCE_CMD_STRICT }, + { 'b', 122, "progress", &progress, SOURCE_CMD_STRICT }, { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, + { 'b', 123, "no-sync", &no_sync, SOURCE_CMD_STRICT }, /* backup options */ { 'b', 133, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, @@ -756,8 +758,8 @@ main(int argc, char *argv[]) } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, - recovery_target_options, - restore_params); + recovery_target_options, + restore_params, no_sync); case VALIDATE_CMD: if (current.backup_id == 0 && target_time == 0 && target_xid == 0 && !target_lsn) { @@ -771,7 +773,8 @@ main(int argc, char *argv[]) /* PITR validation and, optionally, partial validation */ return do_restore_or_validate(current.backup_id, recovery_target_options, - restore_params); + restore_params, + no_sync); case SHOW_CMD: return do_show(instance_name, current.backup_id, show_archive); case DELETE_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d21d8540..d1633617 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -644,7 +644,8 @@ extern char *pg_ptrack_get_block(ConnectionArgs *arguments, /* in restore.c */ extern int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, - pgRestoreParams *params); + pgRestoreParams *params, + bool no_sync); extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); diff --git a/src/restore.c b/src/restore.c index 16f3186c..0c0403b5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -65,7 +65,7 @@ static void pg12_recovery_config(pgBackup *backup, bool add_include); static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path); + const char *pgdata_path, bool no_sync); static void *restore_files_new(void *arg); @@ -118,7 +118,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) */ int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, - pgRestoreParams *params) + pgRestoreParams *params, bool no_sync) { int i = 0; int j = 0; @@ -490,14 +490,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, dest_backup->server_version); restore_chain(dest_backup, parent_chain, dbOid_exclude_list, - params, instance_config.pgdata); + params, instance_config.pgdata, no_sync); /* Create recovery.conf with given recovery target parameters */ create_recovery_conf(target_backup_id, rt, dest_backup, params); } /* cleanup */ - parray_walk(backups, pgBackupFree); /* free backup->files */ + parray_walk(backups, pgBackupFree); /* TODO: free backup->files */ parray_free(backups); parray_free(parent_chain); @@ -512,7 +512,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path) + const char *pgdata_path, bool no_sync) { int i; char control_file[MAXPGPATH]; @@ -524,6 +524,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, restore_files_arg_new *threads_args; bool restore_isok = true; + char pretty_time[20]; time_t start_time, end_time; /* Preparations for actual restoring */ @@ -637,6 +638,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pg_atomic_clear_flag(&file->lock); } + fio_disconnect(); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (restore_files_arg_new *) palloc(sizeof(restore_files_arg_new) * num_threads); @@ -674,47 +677,57 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); if (restore_isok) - elog(INFO, "Backup files are restored, time elapsed: %.0f sec", - difftime(end_time, start_time)); + elog(INFO, "Backup files are restored, time elapsed: %s", pretty_time); else - elog(ERROR, "Backup files restoring failed, time elapsed: %.0f sec", - difftime(end_time, start_time)); + elog(ERROR, "Backup files restoring failed, time elapsed: %s", pretty_time); - - elog(INFO, "Sync restored backup files to disk"); - time(&start_time); - - for (i = 0; i < parray_num(dest_files); i++) + if (no_sync) + elog(WARNING, "Restored files are not synced to disk"); + else { - int out; - char to_fullpath[MAXPGPATH]; - pgFile *dest_file = (pgFile *) parray_get(dest_files, i); + elog(INFO, "Syncing restored files to disk"); + time(&start_time); - if (S_ISDIR(dest_file->mode) || - dest_file->external_dir_num > 0 || - (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) || - (strcmp(DATABASE_MAP, dest_file->rel_path) == 0)) - continue; + for (i = 0; i < parray_num(dest_files); i++) + { + char to_fullpath[MAXPGPATH]; + pgFile *dest_file = (pgFile *) parray_get(dest_files, i); - join_path_components(to_fullpath, pgdata_path, dest_file->rel_path); + if (S_ISDIR(dest_file->mode)) + continue; - /* open destination file */ - out = fio_open(to_fullpath, O_WRONLY | PG_BINARY, FIO_DB_HOST); - if (out < 0) - elog(ERROR, "Cannot open file \"%s\": %s", - to_fullpath, strerror(errno)); + if (params->skip_external_dirs && dest_file->external_dir_num > 0) + continue; - /* sync file */ - if (fio_flush(out) != 0 || fio_close(out) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", - to_fullpath, strerror(errno)); + /* construct fullpath */ + if (dest_file->external_dir_num == 0) + { + if (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) + continue; + if (strcmp(DATABASE_MAP, dest_file->rel_path) == 0) + continue; + join_path_components(to_fullpath, pgdata_path, dest_file->rel_path); + } + else + { + char *external_path = parray_get(external_dirs, dest_file->external_dir_num - 1); + join_path_components(to_fullpath, external_path, dest_file->rel_path); + } + + /* TODO: write test for case: file to be synced is missing */ + if (fio_sync(to_fullpath, FIO_DB_HOST) != 0) + elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno)); + } + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + elog(INFO, "Restored backup files are synced, time elapsed: %s", pretty_time); } - time(&end_time); - elog(INFO, "Restored backup files are synced, time elapsed: %.0f sec", - difftime(end_time, start_time)); - /* cleanup */ pfree(threads); pfree(threads_args); @@ -724,8 +737,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray_walk(dest_files, pgFileFree); parray_free(dest_files); - -// elog(LOG, "Restore of backup %s is completed", base36enc(backup->start_time)); } /* diff --git a/src/utils/file.c b/src/utils/file.c index bae08ef2..b6d1e16d 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -797,6 +797,48 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location } } +/* Sync file to disk */ +int fio_sync(char const* path, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + hdr.cop = FIO_SYNC; + hdr.handle = -1; + hdr.size = path_len; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } + + return 0; + } + else + { + int fd; + + fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); + if (fd < 0) + return -1; + + if (fsync(fd) < 0) + { + close(fd); + return -1; + } + close(fd); + + return 0; + } +} + /* Remove file */ int fio_unlink(char const* path, fio_location location) { @@ -1348,6 +1390,7 @@ void fio_communicate(int in, int out) fio_header hdr; struct stat st; int rc; + int tmp_fd; #ifdef WIN32 SYS_CHECK(setmode(in, _O_BINARY)); @@ -1470,6 +1513,22 @@ void fio_communicate(int in, int out) Assert(hdr.size == sizeof(fio_send_request)); fio_send_pages_impl(fd[hdr.handle], out, (fio_send_request*)buf); break; + case FIO_SYNC: + /* open file and fsync it */ + tmp_fd = open(buf, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); + if (tmp_fd < 0) + hdr.arg = errno; + else + { + if (fsync(tmp_fd) == 0) + hdr.arg = 0; + else + hdr.arg = errno; + } + close(tmp_fd); + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index 70dce425..9b3321a5 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -15,6 +15,7 @@ typedef enum FIO_OPEN, FIO_CLOSE, FIO_WRITE, + FIO_SYNC, FIO_RENAME, FIO_SYMLINK, FIO_UNLINK, @@ -92,6 +93,7 @@ extern int fio_fstat(int fd, struct stat* st); extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); extern void fio_disconnect(void); +extern int fio_sync(char const* path, fio_location location); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, fio_location location);