First successful restore

src/backup.c

@@ -66,10 +66,11 @@ typedef struct
 	 * 0 means there is no error, 1 - there is an error.
 	 */
 	int			ret;
+	parray	   *files_list;
 } StreamThreadArg;
 
 static pthread_t stream_thread;
-static StreamThreadArg stream_thread_arg = {"", NULL, 1};
+static StreamThreadArg stream_thread_arg = {"", NULL, 1, NULL};
 
 static int is_ptrack_enable = false;
 bool is_ptrack_support = false;
@@ -475,6 +476,7 @@ do_backup_instance(void)
 
 	pgBackup   *prev_backup = NULL;
 	parray	   *prev_backup_filelist = NULL;
+	parray	   *xlog_files_list = NULL;
 
 	elog(LOG, "Database backup start");
 
@@ -604,8 +606,12 @@ do_backup_instance(void)
 			elog(ERROR, "Cannot continue backup because stream connect has failed.");
 		}
 
-		/* By default there are some error */
+		if (is_remote_agent)
+			xlog_files_list = parray_new();
+
+        /* By default there are some error */
 		stream_thread_arg.ret = 1;
+		stream_thread_arg.files_list = xlog_files_list;
 
 		pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg);
 	}
@@ -761,18 +767,18 @@ do_backup_instance(void)
 	/* Add archived xlog files into the list of files of this backup */
 	if (stream_wal)
 	{
-		parray	   *xlog_files_list;
-		char		pg_xlog_path[MAXPGPATH];
-
-		/* Scan backup PG_XLOG_DIR */
-		xlog_files_list = parray_new();
-		join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
-		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false);
+		if (xlog_files_list == NULL)
+		{
+			char		pg_xlog_path[MAXPGPATH];
 
+			/* Scan backup PG_XLOG_DIR */
+			xlog_files_list = parray_new();
+			join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
+			dir_list_file(xlog_files_list, pg_xlog_path, false, true, false);
+		}
 		for (i = 0; i < parray_num(xlog_files_list); i++)
 		{
 			pgFile	   *file = (pgFile *) parray_get(xlog_files_list, i);
-
 			if (S_ISREG(file->mode))
 				calc_file_checksum(file);
 			/* Remove file path root prefix*/
@@ -784,7 +790,6 @@ do_backup_instance(void)
 				free(ptr);
 			}
 		}
-
 		/* Add xlog files into the list of backed up files */
 		parray_concat(backup_files_list, xlog_files_list);
 		parray_free(xlog_files_list);
@@ -1874,7 +1879,7 @@ pg_stop_backup(pgBackup *backup)
 			 */
 			if (backup_files_list)
 			{
-				file = pgFileNew(backup_label, true);
+				file = pgFileNew(backup_label, true, FIO_BACKUP_HOST);
 				calc_file_checksum(file);
 				free(file->path);
 				file->path = strdup(PG_BACKUP_LABEL_FILE);
@@ -1917,7 +1922,7 @@ pg_stop_backup(pgBackup *backup)
 
 			if (backup_files_list)
 			{
-				file = pgFileNew(tablespace_map, true);
+				file = pgFileNew(tablespace_map, true, FIO_BACKUP_HOST);
 				if (S_ISREG(file->mode))
 					calc_file_checksum(file);
 				free(file->path);
@@ -2608,7 +2613,7 @@ StreamLog(void *arg)
 		ctl.sysidentifier = NULL;
 
 #if PG_VERSION_NUM >= 100000
-		ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true);
+		ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true, stream_arg->files_list);
 		ctl.replication_slot = replication_slot;
 		ctl.stop_socket = PGINVALID_SOCKET;
 #else

src/data.c

@@ -1331,7 +1331,8 @@ bool
 calc_file_checksum(pgFile *file)
 {
 	int	        in;
-	size_t		read_len = 0;
+	ssize_t		read_len = 0;
+	int         errno_tmp;
 	char		buf[BLCKSZ];
 	struct stat	st;
 	pg_crc32	crc;
@@ -1366,13 +1367,8 @@ calc_file_checksum(pgFile *file)
 			 strerror(errno));
 	}
 
-	for (;;)
+	while  ((read_len = fio_read(in, buf, sizeof(buf))) > 0)
 	{
-		read_len = fio_read(in, buf, sizeof(buf));
-
-		if(read_len == 0)
-			break;
-
 		/* update CRC */
 		COMP_TRADITIONAL_CRC32(crc, buf, read_len);
 
@@ -1380,7 +1376,15 @@ calc_file_checksum(pgFile *file)
 		file->read_size += read_len;
 	}
 
-	/* finish CRC calculation and store into pgFile */
+    errno_tmp = errno;
+    if (read_len < 0)
+    {
+		fio_close(in);
+		elog(ERROR, "cannot read backup mode file \"%s\": %s",
+			 file->path, strerror(errno_tmp));
+	}
+
+    /* finish CRC calculation and store into pgFile */
 	FIN_TRADITIONAL_CRC32(crc);
 	file->crc = crc;
 

src/dir.c

@@ -157,13 +157,13 @@ dir_create_dir(const char *dir, mode_t mode)
 }
 
 pgFile *
-pgFileNew(const char *path, bool omit_symlink)
+pgFileNew(const char *path, bool omit_symlink, fio_location location)
 {
 	struct stat		st;
 	pgFile		   *file;
 
     /* stat the file */
-    if (fio_stat(path, &st, omit_symlink, FIO_BACKUP_HOST) < 0)
+    if (fio_stat(path, &st, omit_symlink, location) < 0)
 	{
 		/* file not found is not an error case */
 		if (errno == ENOENT)
@@ -432,7 +432,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
 		parray_qsort(black_list, BlackListCompare);
 	}
 
-	file = pgFileNew(root, false);
+	file = pgFileNew(root, false, FIO_LOCAL_HOST);
 	if (file == NULL)
 		return;
 
@@ -661,7 +661,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
 
 		join_path_components(child, parent->path, dent->d_name);
 
-		file = pgFileNew(child, omit_symlink);
+		file = pgFileNew(child, omit_symlink, FIO_LOCAL_HOST);
 		if (file == NULL)
 			continue;
 
@@ -795,7 +795,7 @@ list_data_directories(parray *files, const char *path, bool is_root,
 	{
 		pgFile	   *dir;
 
-		dir = pgFileNew(path, false);
+		dir = pgFileNew(path, false, FIO_LOCAL_HOST);
 		parray_append(files, dir);
 	}
 
@@ -935,7 +935,6 @@ create_data_directories(const char *data_dir, const char *backup_dir,
 	size_t		i;
 	char		backup_database_dir[MAXPGPATH],
 				to_path[MAXPGPATH];
-	sleep(30);
 	dirs = parray_new();
 	if (extract_tablespaces)
 	{

src/pg_probackup.h

@@ -505,7 +505,7 @@ extern bool dir_is_empty(const char *path);
 extern bool fileExists(const char *path,  fio_location location);
 extern size_t pgFileSize(const char *path);
 
-extern pgFile *pgFileNew(const char *path, bool omit_symlink);
+extern pgFile *pgFileNew(const char *path, bool omit_symlink, fio_location location);
 extern pgFile *pgFileInit(const char *path);
 extern void pgFileDelete(pgFile *file);
 extern void pgFileFree(void *file);

src/restore.c

@@ -587,14 +587,14 @@ restore_files(void *arg)
 			(arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE
 			|| arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK))
 		{
-			elog(INFO, "The file didn`t change. Skip restore: %s", file->path);
+			elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path);
 			continue;
 		}
 
 		/* Directories were created before */
 		if (S_ISDIR(file->mode))
 		{
-			elog(INFO, "directory, skip");
+			elog(VERBOSE, "directory, skip");
 			continue;
 		}
 
@@ -611,7 +611,7 @@ restore_files(void *arg)
 		 * block and have BackupPageHeader meta information, so we cannot just
 		 * copy the file from backup.
 		 */
-		elog(INFO, "Restoring file %s, is_datafile %i, is_cfs %i",
+		elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i",
 			 file->path, file->is_datafile?1:0, file->is_cfs?1:0);
 		if (file->is_datafile && !file->is_cfs)
 		{

src/walmethods.c

@@ -45,6 +45,7 @@ typedef struct DirectoryMethodData
 	char	   *basedir;
 	int			compression;
 	bool		sync;
+	parray     *files_list;
 } DirectoryMethodData;
 static DirectoryMethodData *dir_data = NULL;
 
@@ -231,6 +232,8 @@ dir_close(Walfile f, WalCloseMethod method)
 
 	if (r == 0)
 	{
+		char const* file_path = NULL;
+
 		/* Build path to the current version of the file */
 		if (method == CLOSE_NORMAL && df->temp_suffix)
 		{
@@ -246,6 +249,7 @@ dir_close(Walfile f, WalCloseMethod method)
 					 dir_data->basedir, df->pathname,
 					 dir_data->compression > 0 ? ".gz" : "");
 			r = durable_rename(tmppath, tmppath2, progname);
+			file_path = tmppath2;
 		}
 		else if (method == CLOSE_UNLINK)
 		{
@@ -263,6 +267,7 @@ dir_close(Walfile f, WalCloseMethod method)
 			 * CLOSE_NO_RENAME. In this case, fsync the file and containing
 			 * directory if sync mode is requested.
 			 */
+			file_path =  df->fullpath;
 			if (dir_data->sync && !is_remote_agent)
 			{
 				r = fsync_fname(df->fullpath, false, progname);
@@ -270,6 +275,12 @@ dir_close(Walfile f, WalCloseMethod method)
 					r = fsync_parent_path(df->fullpath, progname);
 			}
 		}
+		if (file_path && dir_data->files_list)
+		{
+			pgFile*	file = pgFileNew(file_path, false, FIO_BACKUP_HOST);
+			Assert(file != NULL);
+			parray_append(dir_data->files_list, file);
+		}
 	}
 
 	pg_free(df->pathname);
@@ -343,7 +354,7 @@ dir_finish(void)
 
 
 WalWriteMethod *
-CreateWalDirectoryMethod(const char *basedir, int compression, bool sync)
+CreateWalDirectoryMethod(const char *basedir, int compression, bool sync, parray* files_list)
 {
 	WalWriteMethod *method;
 
@@ -362,6 +373,7 @@ CreateWalDirectoryMethod(const char *basedir, int compression, bool sync)
 	dir_data->compression = compression;
 	dir_data->basedir = pg_strdup(basedir);
 	dir_data->sync = sync;
+	dir_data->files_list = files_list;
 
 	return method;
 }

src/walmethods.h

@@ -9,7 +9,7 @@
  *-------------------------------------------------------------------------
  */
 
-
+#include "utils/parray.h"
 typedef void *Walfile;
 
 typedef enum
@@ -86,7 +86,7 @@ struct WalWriteMethod
  *						   not all those required for pg_receivewal)
  */
 WalWriteMethod *CreateWalDirectoryMethod(const char *basedir,
-						 int compression, bool sync);
+										 int compression, bool sync, parray* file_list);
 
 /* Cleanup routines for previously-created methods */
 void		FreeWalDirectoryMethod(void);