diff --git a/src/data.c b/src/data.c index 44f6d337..fc9a92af 100644 --- a/src/data.c +++ b/src/data.c @@ -613,7 +613,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, } if (!fio_is_remote_file(in)) - setbuf(in, in_buffer); + setbuffer(in, in_buffer, STDIO_BUFSIZE); /* open backup file for write */ out = fopen(to_fullpath, PG_BINARY_W); @@ -621,7 +621,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, elog(ERROR, "Cannot open backup file \"%s\": %s", to_fullpath, strerror(errno)); - setbuf(out, out_buffer); + setbuffer(out, out_buffer, STDIO_BUFSIZE); /* update file permission */ if (chmod(to_fullpath, FILE_PERMISSION) == -1) @@ -844,6 +844,10 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char if (tmp_file->write_size == BYTES_INVALID) continue; + /* If file was truncated in intermediate backup, + * it is ok not to truncate it now, because old blocks will be + * overwritten by new blocks from next backup. + */ if (tmp_file->write_size == 0) continue; @@ -859,7 +863,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - setbuf(in, buffer); + setbuffer(in, buffer, STDIO_BUFSIZE); /* * Restore the file. @@ -1149,7 +1153,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - setbuf(in, buffer); + setbuffer(in, buffer, STDIO_BUFSIZE); /* do actual work */ restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); diff --git a/src/help.c b/src/help.c index 5368a335..14ede6dd 100644 --- a/src/help.c +++ b/src/help.c @@ -119,6 +119,7 @@ help_pg_probackup(void) printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [--external-dirs=external-directories-paths]\n")); + printf(_(" [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -156,6 +157,7 @@ help_pg_probackup(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs] [--restore-command=cmdline]\n")); + printf(_(" [--no-sync]\n")); printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); @@ -256,6 +258,7 @@ help_backup(void) printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-E external-directories-paths]\n")); + printf(_(" [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -294,6 +297,7 @@ help_backup(void) printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); + printf(_(" --no-sync do not sync backed up files to disk\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -390,6 +394,7 @@ help_restore(void) printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); printf(_(" [--restore-command=cmdline]\n")); + printf(_(" [--no-sync]\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); @@ -432,6 +437,7 @@ help_restore(void) printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n")); + printf(_(" --no-sync do not sync restored files to disk\n")); printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); diff --git a/src/merge.c b/src/merge.c index 91474fbb..2d286062 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1107,7 +1107,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, if (out == NULL) elog(ERROR, "Cannot open merge target file \"%s\": %s", to_fullpath_tmp1, strerror(errno)); - setbuf(out, buffer); + setbuffer(out, buffer, STDIO_BUFSIZE); /* restore file into temp file */ tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a9132e53..97ef973e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -152,28 +152,29 @@ static void opt_datname_include_list(ConfigOption *opt, const char *arg); /* * Short name should be non-printable ASCII character. + * Use values between 128 and 255. */ static ConfigOption cmd_options[] = { /* directory options */ - { 'b', 120, "help", &help_opt, SOURCE_CMD_STRICT }, + { 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT }, /* common options */ { 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT }, - { 'b', 121, "stream", &stream_wal, SOURCE_CMD_STRICT }, - { 'b', 122, "progress", &progress, SOURCE_CMD_STRICT }, + { 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT }, + { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, - { 'b', 123, "no-sync", &no_sync, SOURCE_CMD_STRICT }, + { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, /* backup options */ - { 'b', 133, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, + { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, - { 'b', 234, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, - { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, - { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, - { 'b', 235, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, - { 'b', 237, "dry-run", &dry_run, SOURCE_CMD_STRICT }, + { 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, + { 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, + { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, + { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, + { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, /* restore options */ { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, @@ -218,7 +219,9 @@ static ConfigOption cmd_options[] = { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, - /* options for backward compatibility */ + /* options for backward compatibility + * TODO: remove in 3.0.0 + */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, diff --git a/src/restore.c b/src/restore.c index 7c6dfece..ce9c650f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -441,7 +441,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time), status2str(dest_backup->status)); /* We ensured that all backups are valid, now restore if required - * TODO: before restore - lock entire parent chain */ if (params->is_restore) { @@ -555,7 +554,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, else backup->files = dest_files; - /* this sorting is important */ + /* + * this sorting is important, because we rely on it to find + * destination file in intermediate backups file lists + * using bsearch. + */ parray_qsort(backup->files, pgFileCompareRelPathWithExternal); } @@ -622,8 +625,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) * num_threads); - - /* Restore files into target directory */ if (dest_backup->stream) dest_bytes = dest_backup->pgdata_bytes + dest_backup->wal_bytes; else @@ -633,6 +634,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(INFO, "Start restoring backup files. PGDATA size: %s", pretty_dest_bytes); time(&start_time); thread_interrupted = false; + + /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { restore_files_arg *arg = &(threads_args[i]); @@ -851,7 +854,7 @@ restore_files(void *arg) goto done; if (!fio_is_remote_file(out)) - setbuf(out, buffer); + setbuffer(out, buffer, STDIO_BUFSIZE); /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) diff --git a/src/utils/file.c b/src/utils/file.c index 73fd780f..031730ad 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -549,6 +549,7 @@ int fio_pread(FILE* f, void* buf, off_t offs) } else { + /* For local file, opened by fopen, we should use stdio operations */ int rc; rc = fseek(f, offs, SEEK_SET); diff --git a/src/validate.c b/src/validate.c index 4f16edc6..167f86b9 100644 --- a/src/validate.c +++ b/src/validate.c @@ -264,6 +264,7 @@ pgBackupValidateFiles(void *arg) */ if (file->write_size == BYTES_INVALID) { + /* TODO: lookup corresponding merge bug */ if (arguments->backup_mode == BACKUP_MODE_FULL) { /* It is illegal for file in FULL backup to have BYTES_INVALID */ @@ -276,10 +277,11 @@ pgBackupValidateFiles(void *arg) continue; } - /* no point in trying to open empty or non-changed files */ - if (file->write_size <= 0) + /* no point in trying to open empty file */ + if (file->write_size == 0) continue; + /* TODO: it is redundant to check file existence using stat */ if (stat(file->path, &st) == -1) { if (errno == ENOENT)