[Issue #185] Big ugly kludge to make prefetch to work in case of multi-timeline recovery

src/archive.c

@@ -1129,8 +1129,18 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg,
 			/* We`ve failed to satisfy current request from prefetch directory,
 			 * therefore we can discard its content, since it may be corrupted or
 			 * contain stale files.
+			 *
+			 * UPDATE: cannot do that:
+			 * https://www.postgresql.org/message-id/dd6690b0-ec03-6b3c-6fac-c963f91f87a7%40postgrespro.ru
 			 */
-			rmtree(prefetch_dir, false);
+
+			//rmtree(prefetch_dir, false);
+			/* count the number of files in prefetch directory ... */
+			if (count_files_in_dir(prefetch_dir) > batch_size)
+				/* ... if it exeeds batch size,
+				 * then we assume that prefetch directory contain garbage
+				 */
+				rmtree(prefetch_dir, false);
 
 			/* prefetch files */
 			n_prefetched = run_wal_prefetch(prefetch_dir, instance->arclog_path,
@@ -1149,7 +1159,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg,
 			{
 				/* prefetch failed again, discard it */
 				n_prefetched = 0;
-				rmtree(prefetch_dir, false);
+//				rmtree(prefetch_dir, false);
 			}
 		}
 	}

src/dir.c

@@ -1626,6 +1626,40 @@ dir_is_empty(const char *path, fio_location location)
 	return true;
 }
 
+/*
+ * Check if directory empty.
+ */
+uint32
+count_files_in_dir(const char *path)
+{
+	DIR		   *dir;
+	struct dirent *dir_ent;
+	uint32 n_files = 0;
+
+	dir = opendir(path);
+	if (dir == NULL)
+	{
+		/* Directory in path doesn't exist */
+		if (errno == ENOENT)
+			return n_files;
+		elog(ERROR, "Cannot open directory \"%s\": %s", path, strerror(errno));
+	}
+
+	while ((dir_ent = readdir(dir)))
+	{
+		/* Skip entries point current dir or parent dir */
+		if (strcmp(dir_ent->d_name, ".") == 0 ||
+			strcmp(dir_ent->d_name, "..") == 0)
+			continue;
+
+		n_files++;
+	}
+
+	fio_closedir(dir);
+
+	return n_files;
+}
+
 /*
  * Return true if the path is a existing regular file.
  */

src/pg_probackup.h

@@ -855,6 +855,7 @@ extern bool backup_contains_external(const char *dir, parray *dirs_list);
 
 extern int dir_create_dir(const char *path, mode_t mode);
 extern bool dir_is_empty(const char *path, fio_location location);
+extern uint32 count_files_in_dir(const char *path);
 
 extern bool fileExists(const char *path, fio_location location);
 extern size_t pgFileSize(const char *path);

src/restore.c

@@ -988,7 +988,7 @@ create_recovery_conf(time_t backup_id,
 		{
 			/* default cmdline, ok for local restore */
 			sprintf(restore_command_guc, "%s archive-get -B %s --instance %s "
-					"--wal-file-path=%%p --wal-file-name=%%f",
+					"--wal-file-path=%%p --wal-file-name=%%f --batch-size=20",
 					PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME,
 					backup_path, instance_name);