diff --git a/src/backup.c b/src/backup.c index 9593e6ab..2dc6af1e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -66,10 +66,11 @@ typedef struct * 0 means there is no error, 1 - there is an error. */ int ret; + parray *files_list; } StreamThreadArg; static pthread_t stream_thread; -static StreamThreadArg stream_thread_arg = {"", NULL, 1}; +static StreamThreadArg stream_thread_arg = {"", NULL, 1, NULL}; static int is_ptrack_enable = false; bool is_ptrack_support = false; @@ -475,6 +476,7 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; + parray *xlog_files_list = NULL; elog(LOG, "Database backup start"); @@ -604,8 +606,12 @@ do_backup_instance(void) elog(ERROR, "Cannot continue backup because stream connect has failed."); } - /* By default there are some error */ + if (is_remote_agent) + xlog_files_list = parray_new(); + + /* By default there are some error */ stream_thread_arg.ret = 1; + stream_thread_arg.files_list = xlog_files_list; pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } @@ -761,18 +767,18 @@ do_backup_instance(void) /* Add archived xlog files into the list of files of this backup */ if (stream_wal) { - parray *xlog_files_list; - char pg_xlog_path[MAXPGPATH]; - - /* Scan backup PG_XLOG_DIR */ - xlog_files_list = parray_new(); - join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + if (xlog_files_list == NULL) + { + char pg_xlog_path[MAXPGPATH]; + /* Scan backup PG_XLOG_DIR */ + xlog_files_list = parray_new(); + join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + } for (i = 0; i < parray_num(xlog_files_list); i++) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); - if (S_ISREG(file->mode)) calc_file_checksum(file); /* Remove file path root prefix*/ @@ -784,7 +790,6 @@ do_backup_instance(void) free(ptr); } } - /* Add xlog files into the list of backed up files */ parray_concat(backup_files_list, xlog_files_list); parray_free(xlog_files_list); @@ -1874,7 +1879,7 @@ pg_stop_backup(pgBackup *backup) */ if (backup_files_list) { - file = pgFileNew(backup_label, true); + file = pgFileNew(backup_label, true, FIO_BACKUP_HOST); calc_file_checksum(file); free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); @@ -1917,7 +1922,7 @@ pg_stop_backup(pgBackup *backup) if (backup_files_list) { - file = pgFileNew(tablespace_map, true); + file = pgFileNew(tablespace_map, true, FIO_BACKUP_HOST); if (S_ISREG(file->mode)) calc_file_checksum(file); free(file->path); @@ -2608,7 +2613,7 @@ StreamLog(void *arg) ctl.sysidentifier = NULL; #if PG_VERSION_NUM >= 100000 - ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); + ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true, stream_arg->files_list); ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; #else diff --git a/src/data.c b/src/data.c index eb2230b6..e3725395 100644 --- a/src/data.c +++ b/src/data.c @@ -1331,7 +1331,8 @@ bool calc_file_checksum(pgFile *file) { int in; - size_t read_len = 0; + ssize_t read_len = 0; + int errno_tmp; char buf[BLCKSZ]; struct stat st; pg_crc32 crc; @@ -1366,13 +1367,8 @@ calc_file_checksum(pgFile *file) strerror(errno)); } - for (;;) + while ((read_len = fio_read(in, buf, sizeof(buf))) > 0) { - read_len = fio_read(in, buf, sizeof(buf)); - - if(read_len == 0) - break; - /* update CRC */ COMP_TRADITIONAL_CRC32(crc, buf, read_len); @@ -1380,7 +1376,15 @@ calc_file_checksum(pgFile *file) file->read_size += read_len; } - /* finish CRC calculation and store into pgFile */ + errno_tmp = errno; + if (read_len < 0) + { + fio_close(in); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* finish CRC calculation and store into pgFile */ FIN_TRADITIONAL_CRC32(crc); file->crc = crc; diff --git a/src/dir.c b/src/dir.c index bf5ec27f..bf5e5008 100644 --- a/src/dir.c +++ b/src/dir.c @@ -157,13 +157,13 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, bool omit_symlink) +pgFileNew(const char *path, bool omit_symlink, fio_location location) { struct stat st; pgFile *file; /* stat the file */ - if (fio_stat(path, &st, omit_symlink, FIO_BACKUP_HOST) < 0) + if (fio_stat(path, &st, omit_symlink, location) < 0) { /* file not found is not an error case */ if (errno == ENOENT) @@ -432,7 +432,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, false); + file = pgFileNew(root, false, FIO_LOCAL_HOST); if (file == NULL) return; @@ -661,7 +661,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, join_path_components(child, parent->path, dent->d_name); - file = pgFileNew(child, omit_symlink); + file = pgFileNew(child, omit_symlink, FIO_LOCAL_HOST); if (file == NULL) continue; @@ -795,7 +795,7 @@ list_data_directories(parray *files, const char *path, bool is_root, { pgFile *dir; - dir = pgFileNew(path, false); + dir = pgFileNew(path, false, FIO_LOCAL_HOST); parray_append(files, dir); } @@ -935,7 +935,6 @@ create_data_directories(const char *data_dir, const char *backup_dir, size_t i; char backup_database_dir[MAXPGPATH], to_path[MAXPGPATH]; - sleep(30); dirs = parray_new(); if (extract_tablespaces) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b4bd8edf..20399734 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -505,7 +505,7 @@ extern bool dir_is_empty(const char *path); extern bool fileExists(const char *path, fio_location location); extern size_t pgFileSize(const char *path); -extern pgFile *pgFileNew(const char *path, bool omit_symlink); +extern pgFile *pgFileNew(const char *path, bool omit_symlink, fio_location location); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); diff --git a/src/restore.c b/src/restore.c index 2cf291d8..51d9afc3 100644 --- a/src/restore.c +++ b/src/restore.c @@ -587,14 +587,14 @@ restore_files(void *arg) (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) { - elog(INFO, "The file didn`t change. Skip restore: %s", file->path); + elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); continue; } /* Directories were created before */ if (S_ISDIR(file->mode)) { - elog(INFO, "directory, skip"); + elog(VERBOSE, "directory, skip"); continue; } @@ -611,7 +611,7 @@ restore_files(void *arg) * block and have BackupPageHeader meta information, so we cannot just * copy the file from backup. */ - elog(INFO, "Restoring file %s, is_datafile %i, is_cfs %i", + elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0); if (file->is_datafile && !file->is_cfs) { diff --git a/src/walmethods.c b/src/walmethods.c index a179945f..39f4f990 100644 --- a/src/walmethods.c +++ b/src/walmethods.c @@ -45,6 +45,7 @@ typedef struct DirectoryMethodData char *basedir; int compression; bool sync; + parray *files_list; } DirectoryMethodData; static DirectoryMethodData *dir_data = NULL; @@ -231,6 +232,8 @@ dir_close(Walfile f, WalCloseMethod method) if (r == 0) { + char const* file_path = NULL; + /* Build path to the current version of the file */ if (method == CLOSE_NORMAL && df->temp_suffix) { @@ -246,6 +249,7 @@ dir_close(Walfile f, WalCloseMethod method) dir_data->basedir, df->pathname, dir_data->compression > 0 ? ".gz" : ""); r = durable_rename(tmppath, tmppath2, progname); + file_path = tmppath2; } else if (method == CLOSE_UNLINK) { @@ -263,6 +267,7 @@ dir_close(Walfile f, WalCloseMethod method) * CLOSE_NO_RENAME. In this case, fsync the file and containing * directory if sync mode is requested. */ + file_path = df->fullpath; if (dir_data->sync && !is_remote_agent) { r = fsync_fname(df->fullpath, false, progname); @@ -270,6 +275,12 @@ dir_close(Walfile f, WalCloseMethod method) r = fsync_parent_path(df->fullpath, progname); } } + if (file_path && dir_data->files_list) + { + pgFile* file = pgFileNew(file_path, false, FIO_BACKUP_HOST); + Assert(file != NULL); + parray_append(dir_data->files_list, file); + } } pg_free(df->pathname); @@ -343,7 +354,7 @@ dir_finish(void) WalWriteMethod * -CreateWalDirectoryMethod(const char *basedir, int compression, bool sync) +CreateWalDirectoryMethod(const char *basedir, int compression, bool sync, parray* files_list) { WalWriteMethod *method; @@ -362,6 +373,7 @@ CreateWalDirectoryMethod(const char *basedir, int compression, bool sync) dir_data->compression = compression; dir_data->basedir = pg_strdup(basedir); dir_data->sync = sync; + dir_data->files_list = files_list; return method; } diff --git a/src/walmethods.h b/src/walmethods.h index 76eb249b..b4dd40c5 100644 --- a/src/walmethods.h +++ b/src/walmethods.h @@ -9,7 +9,7 @@ *------------------------------------------------------------------------- */ - +#include "utils/parray.h" typedef void *Walfile; typedef enum @@ -86,7 +86,7 @@ struct WalWriteMethod * not all those required for pg_receivewal) */ WalWriteMethod *CreateWalDirectoryMethod(const char *basedir, - int compression, bool sync); + int compression, bool sync, parray* file_list); /* Cleanup routines for previously-created methods */ void FreeWalDirectoryMethod(void);