db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-02-07 14:18:17 +02:00
Code Issues Releases Activity

[Issue #169] fio_sync() added

Browse Source
This commit is contained in:
Grigory Smolkin 2020-01-26 21:39:13 +03:00
parent 577e8763bc
commit bf5b8f6b48
5 changed files with 121 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/pg_probackup.c
View File

@ -68,6 +68,7 @@ static char *backup_id_string = NULL;
int num_threads = 1;
bool stream_wal = false;
bool progress = false;
bool no_sync = false;
#if PG_VERSION_NUM >= 100000
char *replication_slot = NULL;
#endif
@ -155,13 +156,14 @@ static void opt_datname_include_list(ConfigOption *opt, const char *arg);
static ConfigOption cmd_options[] =
{
/* directory options */
{ 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT },
{ 'b', 120, "help", &help_opt, SOURCE_CMD_STRICT },
{ 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT },
/* common options */
{ 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT },
{ 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT },
{ 'b', 132, "progress", &progress, SOURCE_CMD_STRICT },
{ 'b', 121, "stream", &stream_wal, SOURCE_CMD_STRICT },
{ 'b', 122, "progress", &progress, SOURCE_CMD_STRICT },
{ 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT },
{ 'b', 123, "no-sync", &no_sync, SOURCE_CMD_STRICT },
/* backup options */
{ 'b', 133, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT },
{ 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT },
@ -756,8 +758,8 @@ main(int argc, char *argv[])
}
case RESTORE_CMD:
return do_restore_or_validate(current.backup_id,
recovery_target_options,
restore_params);
recovery_target_options,
restore_params, no_sync);
case VALIDATE_CMD:
if (current.backup_id == 0 && target_time == 0 && target_xid == 0 && !target_lsn)
{
@ -771,7 +773,8 @@ main(int argc, char *argv[])
/* PITR validation and, optionally, partial validation */
return do_restore_or_validate(current.backup_id,
recovery_target_options,
restore_params);
restore_params,
no_sync);
case SHOW_CMD:
return do_show(instance_name, current.backup_id, show_archive);
case DELETE_CMD:

3
src/pg_probackup.h
View File

@ -644,7 +644,8 @@ extern char *pg_ptrack_get_block(ConnectionArgs *arguments,
/* in restore.c */
extern int do_restore_or_validate(time_t target_backup_id,
pgRecoveryTarget *rt,
pgRestoreParams *params);
pgRestoreParams *params,
bool no_sync);
extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup);
extern bool satisfy_recovery_target(const pgBackup *backup,
const pgRecoveryTarget *rt);

87
src/restore.c
View File

@ -65,7 +65,7 @@ static void pg12_recovery_config(pgBackup *backup, bool add_include);
static void restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray *dbOid_exclude_list, pgRestoreParams *params,
const char *pgdata_path);
const char *pgdata_path, bool no_sync);
static void *restore_files_new(void *arg);
@ -118,7 +118,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup)
*/
int
do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
pgRestoreParams *params)
pgRestoreParams *params, bool no_sync)
{
int i = 0;
int j = 0;
@ -490,14 +490,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
dest_backup->server_version);
restore_chain(dest_backup, parent_chain, dbOid_exclude_list,
params, instance_config.pgdata);
params, instance_config.pgdata, no_sync);
/* Create recovery.conf with given recovery target parameters */
create_recovery_conf(target_backup_id, rt, dest_backup, params);
}
/* cleanup */
parray_walk(backups, pgBackupFree); /* free backup->files */
parray_walk(backups, pgBackupFree); /* TODO: free backup->files */
parray_free(backups);
parray_free(parent_chain);
@ -512,7 +512,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
void
restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray *dbOid_exclude_list, pgRestoreParams *params,
const char *pgdata_path)
const char *pgdata_path, bool no_sync)
{
int i;
char control_file[MAXPGPATH];
@ -524,6 +524,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
restore_files_arg_new *threads_args;
bool restore_isok = true;
char pretty_time[20];
time_t start_time, end_time;
/* Preparations for actual restoring */
@ -637,6 +638,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
pg_atomic_clear_flag(&file->lock);
}
fio_disconnect();
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
threads_args = (restore_files_arg_new *) palloc(sizeof(restore_files_arg_new) *
num_threads);
@ -674,47 +677,57 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
}
time(&end_time);
pretty_time_interval(difftime(end_time, start_time),
pretty_time, lengthof(pretty_time));
if (restore_isok)
elog(INFO, "Backup files are restored, time elapsed: %.0f sec",
difftime(end_time, start_time));
elog(INFO, "Backup files are restored, time elapsed: %s", pretty_time);
else
elog(ERROR, "Backup files restoring failed, time elapsed: %.0f sec",
difftime(end_time, start_time));
elog(ERROR, "Backup files restoring failed, time elapsed: %s", pretty_time);
elog(INFO, "Sync restored backup files to disk");
time(&start_time);
for (i = 0; i < parray_num(dest_files); i++)
if (no_sync)
elog(WARNING, "Restored files are not synced to disk");
else
{
int out;
char to_fullpath[MAXPGPATH];
pgFile *dest_file = (pgFile *) parray_get(dest_files, i);
elog(INFO, "Syncing restored files to disk");
time(&start_time);
if (S_ISDIR(dest_file->mode) ||
dest_file->external_dir_num > 0 ||
(strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) ||
(strcmp(DATABASE_MAP, dest_file->rel_path) == 0))
continue;
for (i = 0; i < parray_num(dest_files); i++)
{
char to_fullpath[MAXPGPATH];
pgFile *dest_file = (pgFile *) parray_get(dest_files, i);
join_path_components(to_fullpath, pgdata_path, dest_file->rel_path);
if (S_ISDIR(dest_file->mode))
continue;
/* open destination file */
out = fio_open(to_fullpath, O_WRONLY | PG_BINARY, FIO_DB_HOST);
if (out < 0)
elog(ERROR, "Cannot open file \"%s\": %s",
to_fullpath, strerror(errno));
if (params->skip_external_dirs && dest_file->external_dir_num > 0)
continue;
/* sync file */
if (fio_flush(out) != 0 || fio_close(out) != 0)
elog(ERROR, "Cannot sync file \"%s\": %s",
to_fullpath, strerror(errno));
/* construct fullpath */
if (dest_file->external_dir_num == 0)
{
if (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0)
continue;
if (strcmp(DATABASE_MAP, dest_file->rel_path) == 0)
continue;
join_path_components(to_fullpath, pgdata_path, dest_file->rel_path);
}
else
{
char *external_path = parray_get(external_dirs, dest_file->external_dir_num - 1);
join_path_components(to_fullpath, external_path, dest_file->rel_path);
}
/* TODO: write test for case: file to be synced is missing */
if (fio_sync(to_fullpath, FIO_DB_HOST) != 0)
elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno));
}
time(&end_time);
pretty_time_interval(difftime(end_time, start_time),
pretty_time, lengthof(pretty_time));
elog(INFO, "Restored backup files are synced, time elapsed: %s", pretty_time);
}
time(&end_time);
elog(INFO, "Restored backup files are synced, time elapsed: %.0f sec",
difftime(end_time, start_time));
/* cleanup */
pfree(threads);
pfree(threads_args);
@ -724,8 +737,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray_walk(dest_files, pgFileFree);
parray_free(dest_files);
// elog(LOG, "Restore of backup %s is completed", base36enc(backup->start_time));
}
/*

59
src/utils/file.c
View File

@ -797,6 +797,48 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location
}
}
/* Sync file to disk */
int fio_sync(char const* path, fio_location location)
{
if (fio_is_remote(location))
{
fio_header hdr;
size_t path_len = strlen(path) + 1;
hdr.cop = FIO_SYNC;
hdr.handle = -1;
hdr.size = path_len;
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len);
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
if (hdr.arg != 0)
{
errno = hdr.arg;
return -1;
}
return 0;
}
else
{
int fd;
fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS);
if (fd < 0)
return -1;
if (fsync(fd) < 0)
{
close(fd);
return -1;
}
close(fd);
return 0;
}
}
/* Remove file */
int fio_unlink(char const* path, fio_location location)
{
@ -1348,6 +1390,7 @@ void fio_communicate(int in, int out)
fio_header hdr;
struct stat st;
int rc;
int tmp_fd;
#ifdef WIN32
SYS_CHECK(setmode(in, _O_BINARY));
@ -1470,6 +1513,22 @@ void fio_communicate(int in, int out)
Assert(hdr.size == sizeof(fio_send_request));
fio_send_pages_impl(fd[hdr.handle], out, (fio_send_request*)buf);
break;
case FIO_SYNC:
/* open file and fsync it */
tmp_fd = open(buf, O_WRONLY | PG_BINARY, FILE_PERMISSIONS);
if (tmp_fd < 0)
hdr.arg = errno;
else
{
if (fsync(tmp_fd) == 0)
hdr.arg = 0;
else
hdr.arg = errno;
}
close(tmp_fd);
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
break;
default:
Assert(false);
}

2
src/utils/file.h
View File

@ -15,6 +15,7 @@ typedef enum
FIO_OPEN,
FIO_CLOSE,
FIO_WRITE,
FIO_SYNC,
FIO_RENAME,
FIO_SYMLINK,
FIO_UNLINK,
@ -92,6 +93,7 @@ extern int fio_fstat(int fd, struct stat* st);
extern int fio_truncate(int fd, off_t size);
extern int fio_close(int fd);
extern void fio_disconnect(void);
extern int fio_sync(char const* path, fio_location location);
extern int fio_rename(char const* old_path, char const* new_path, fio_location location);
extern int fio_symlink(char const* target, char const* link_path, fio_location location);