change datafile validation algorithm:

- validate file block by block by default, not only in case of file-level checksum corruption;
    - add an option: --skip-block-validation to disable this behaviour;
    - calculate file checksum at the same time as validate blocks;

src/data.c

@@ -1601,12 +1601,14 @@ validate_one_page(Page page, pgFile *file,
 
 /* Valiate pages of datafile in backup one by one */
 bool
-check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
-				 uint32 backup_version)
+check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
+				 uint32 checksum_version, uint32 backup_version)
 {
 	size_t		read_len = 0;
 	bool		is_valid = true;
 	FILE		*in;
+	pg_crc32	crc;
+	bool use_crc32c = (backup_version <= 20021);
 
 	elog(VERBOSE, "validate relation blocks for file %s", file->name);
 
@@ -1623,6 +1625,9 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
 			 file->path, strerror(errno));
 	}
 
+	/* calc CRC of backup file */
+	INIT_FILE_CRC32(use_crc32c, crc);
+
 	/* read and validate pages one by one */
 	while (true)
 	{
@@ -1647,6 +1652,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
 					 blknum, file->path, strerror(errno_tmp));
 		}
 
+		COMP_FILE_CRC32(use_crc32c, crc, &header, read_len);
+
 		if (header.block < blknum)
 			elog(ERROR, "backup is broken at file->path %s block %u",
 				 file->path, blknum);
@@ -1668,6 +1675,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
 			elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d",
 				blknum, file->path, read_len, header.compressed_size);
 
+		COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len);
+
 		if (header.compressed_size != BLCKSZ
 			|| page_may_be_compressed(compressed_page.data, file->compress_alg,
 									  backup_version))
@@ -1706,5 +1715,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
 		}
 	}
 
+	FIN_FILE_CRC32(use_crc32c, crc);
+
+	if (crc != file->crc)
+	{
+		elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
+				file->path, file->crc, crc);
+		is_valid = false;
+	}
+
 	return is_valid;
 }

src/dir.c

@@ -267,28 +267,6 @@ pgFileGetCRC(const char *file_path, bool use_crc32c)
 	size_t		len;
 	int			errno_tmp;
 
-#define INIT_FILE_CRC32(crc) \
-do { \
-	if (use_crc32c) \
-		INIT_CRC32C(crc); \
-	else \
-		INIT_TRADITIONAL_CRC32(crc); \
-} while (0)
-#define COMP_FILE_CRC32(crc, data, len) \
-do { \
-	if (use_crc32c) \
-		COMP_CRC32C((crc), (data), (len)); \
-	else \
-		COMP_TRADITIONAL_CRC32(crc, data, len); \
-} while (0)
-#define FIN_FILE_CRC32(crc) \
-do { \
-	if (use_crc32c) \
-		FIN_CRC32C(crc); \
-	else \
-		FIN_TRADITIONAL_CRC32(crc); \
-} while (0)
-
 	/* open file in binary read mode */
 	fp = fopen(file_path, PG_BINARY_R);
 	if (fp == NULL)
@@ -296,20 +274,20 @@ do { \
 			file_path, strerror(errno));
 
 	/* calc CRC of backup file */
-	INIT_FILE_CRC32(crc);
+	INIT_FILE_CRC32(use_crc32c, crc);
 	while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf))
 	{
 		if (interrupted)
 			elog(ERROR, "interrupted during CRC calculation");
-		COMP_FILE_CRC32(crc, buf, len);
+		COMP_FILE_CRC32(use_crc32c, crc, buf, len);
 	}
 	errno_tmp = errno;
 	if (!feof(fp))
 		elog(WARNING, "cannot read \"%s\": %s", file_path,
 			strerror(errno_tmp));
 	if (len > 0)
-		COMP_FILE_CRC32(crc, buf, len);
-	FIN_FILE_CRC32(crc);
+		COMP_FILE_CRC32(use_crc32c, crc, buf, len);
+	FIN_FILE_CRC32(use_crc32c, crc);
 
 	fclose(fp);
 

src/help.c

@@ -118,6 +118,7 @@ help_pg_probackup(void)
 	printf(_("                 [--master-db=db_name] [--master-host=host_name]\n"));
 	printf(_("                 [--master-port=port] [--master-user=user_name]\n"));
 	printf(_("                 [--replica-timeout=timeout]\n"));
+	printf(_("                 [--skip-block-validation]\n"));
 
 	printf(_("\n  %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
 	printf(_("                 [-D pgdata-path] [-i backup-id] [--progress]\n"));
@@ -127,12 +128,14 @@ help_pg_probackup(void)
 	printf(_("                 [--recovery-target-action=pause|promote|shutdown]\n"));
 	printf(_("                 [--restore-as-replica]\n"));
 	printf(_("                 [--no-validate]\n"));
+	printf(_("                 [--skip-block-validation]\n"));
 
 	printf(_("\n  %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME);
 	printf(_("                 [-i backup-id] [--progress]\n"));
 	printf(_("                 [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n"));
 	printf(_("                 [--recovery-target-name=target-name]\n"));
 	printf(_("                 [--timeline=timeline]\n"));
+	printf(_("                 [--skip-block-validation]\n"));
 
 	printf(_("\n  %s show -B backup-path\n"), PROGRAM_NAME);
 	printf(_("                 [--instance=instance_name [-i backup-id]]\n"));
@@ -203,7 +206,8 @@ help_backup(void)
 	printf(_("                 [-w --no-password] [-W --password]\n"));
 	printf(_("                 [--master-db=db_name] [--master-host=host_name]\n"));
 	printf(_("                 [--master-port=port] [--master-user=user_name]\n"));
-	printf(_("                 [--replica-timeout=timeout]\n\n"));
+	printf(_("                 [--replica-timeout=timeout]\n"));
+	printf(_("                 [--skip-block-validation]\n\n"));
 
 	printf(_("  -B, --backup-path=backup-path    location of the backup storage area\n"));
 	printf(_("  -b, --backup-mode=backup-mode    backup mode=FULL|PAGE|DELTA|PTRACK\n"));
@@ -215,6 +219,7 @@ help_backup(void)
 	printf(_("  -j, --threads=NUM                number of parallel threads\n"));
 	printf(_("      --archive-timeout=timeout    wait timeout for WAL segment archiving (default: 5min)\n"));
 	printf(_("      --progress                   show progress\n"));
+	printf(_("      --skip-block-validation      set to validate only file-level checksum\n"));
 
 	printf(_("\n  Logging options:\n"));
 	printf(_("      --log-level-console=log-level-console\n"));
@@ -279,6 +284,7 @@ help_restore(void)
 	printf(_("                 [--immediate] [--recovery-target-name=target-name]\n"));
 	printf(_("                 [--recovery-target-action=pause|promote|shutdown]\n"));
 	printf(_("                 [--restore-as-replica] [--no-validate]\n\n"));
+	printf(_("                 [--skip-block-validation]\n\n"));
 
 	printf(_("  -B, --backup-path=backup-path    location of the backup storage area\n"));
 	printf(_("      --instance=instance_name     name of the instance\n"));
@@ -305,6 +311,7 @@ help_restore(void)
 	printf(_("  -R, --restore-as-replica         write a minimal recovery.conf in the output directory\n"));
 	printf(_("                                   to ease setting up a standby server\n"));
 	printf(_("      --no-validate                disable backup validation during restore\n"));
+	printf(_("      --skip-block-validation      set to validate only file-level checksum\n"));
 
 	printf(_("\n  Logging options:\n"));
 	printf(_("      --log-level-console=log-level-console\n"));
@@ -335,6 +342,7 @@ help_validate(void)
 	printf(_("                 [-i backup-id] [--progress]\n"));
 	printf(_("                 [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n"));
 	printf(_("                 [--timeline=timeline]\n\n"));
+	printf(_("                 [--skip-block-validation]\n\n"));
 
 	printf(_("  -B, --backup-path=backup-path    location of the backup storage area\n"));
 	printf(_("      --instance=instance_name     name of the instance\n"));
@@ -348,6 +356,7 @@ help_validate(void)
 	printf(_("      --timeline=timeline          recovering into a particular timeline\n"));
 	printf(_("      --recovery-target-name=target-name\n"));
 	printf(_("                                   the named restore point to which recovery will proceed\n"));
+	printf(_("      --skip-block-validation      set to validate only file-level checksum\n"));
 
 	printf(_("\n  Logging options:\n"));
 	printf(_("      --log-level-console=log-level-console\n"));

src/pg_probackup.c

@@ -89,6 +89,8 @@ static pgRecoveryTarget *recovery_target_options = NULL;
 bool restore_as_replica = false;
 bool restore_no_validate = false;
 
+bool skip_block_validation = false;
+
 /* delete options */
 bool		delete_wal = false;
 bool		delete_expired = false;
@@ -179,6 +181,7 @@ static pgut_option options[] =
 	{ 'b', 'R', "restore-as-replica",	&restore_as_replica,	SOURCE_CMDLINE },
 	{ 'b', 27, "no-validate",			&restore_no_validate,	SOURCE_CMDLINE },
 	{ 's', 28, "lsn",					&target_lsn,		SOURCE_CMDLINE },
+	{ 'b', 29, "skip-block-validation", &skip_block_validation,	SOURCE_CMDLINE },
 	/* delete options */
 	{ 'b', 130, "wal",					&delete_wal,		SOURCE_CMDLINE },
 	{ 'b', 131, "expired",				&delete_expired,	SOURCE_CMDLINE },

src/pg_probackup.h

@@ -65,6 +65,28 @@ typedef enum CompressAlg
 	ZLIB_COMPRESS,
 } CompressAlg;
 
+#define INIT_FILE_CRC32(use_crc32c, crc) \
+do { \
+	if (use_crc32c) \
+		INIT_CRC32C(crc); \
+	else \
+		INIT_TRADITIONAL_CRC32(crc); \
+} while (0)
+#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \
+do { \
+	if (use_crc32c) \
+		COMP_CRC32C((crc), (data), (len)); \
+	else \
+		COMP_TRADITIONAL_CRC32(crc, data, len); \
+} while (0)
+#define FIN_FILE_CRC32(use_crc32c, crc) \
+do { \
+	if (use_crc32c) \
+		FIN_CRC32C(crc); \
+	else \
+		FIN_TRADITIONAL_CRC32(crc); \
+} while (0)
+
 /* Information about single file (or dir) in backup */
 typedef struct pgFile
 {
@@ -339,6 +361,7 @@ extern bool exclusive_backup;
 
 /* restore options */
 extern bool restore_as_replica;
+extern bool skip_block_validation;
 
 /* delete options */
 extern bool		delete_wal;
@@ -527,9 +550,9 @@ extern void get_wal_file(const char *from_path, const char *to_path);
 
 extern bool calc_file_checksum(pgFile *file);
 
-extern bool check_file_pages(pgFile* file, XLogRecPtr stop_lsn,
+extern bool check_file_pages(pgFile* file,
+							 XLogRecPtr stop_lsn,
 							 uint32 checksum_version, uint32 backup_version);
-
 /* parsexlog.c */
 extern void extractPageMap(const char *archivedir,
 						   TimeLineID tli, uint32 seg_size,

src/validate.c

@@ -208,32 +208,42 @@ pgBackupValidateFiles(void *arg)
 		}
 
 		/*
-		 * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
-		 * use CRC-32.
-		 *
-		 * pg_control stores its content and checksum of the content, calculated
-		 * using CRC-32C. If we calculate checksum of the whole pg_control using
-		 * CRC-32C we get same checksum constantly. It might be because of the
-		 * CRC-32C algorithm.
-		 * To avoid this problem we need to use different algorithm, CRC-32 in
-		 * this case.
+		 * If option skip-block-validation is set, compute only file-level CRC for
+		 * datafiles, otherwise check them block by block.
 		 */
-		crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021);
-		if (crc != file->crc)
+		if (!file->is_datafile || skip_block_validation)
 		{
-			elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
-					file->path, file->crc, crc);
-			arguments->corrupted = true;
-
-			/* validate relation blocks */
-			if (file->is_datafile)
+			/*
+			 * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
+			 * use CRC-32.
+			 *
+			 * pg_control stores its content and checksum of the content, calculated
+			 * using CRC-32C. If we calculate checksum of the whole pg_control using
+			 * CRC-32C we get same checksum constantly. It might be because of the
+			 * CRC-32C algorithm.
+			 * To avoid this problem we need to use different algorithm, CRC-32 in
+			 * this case.
+			 */
+			crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021);
+			if (crc != file->crc)
 			{
-				if (!check_file_pages(file, arguments->stop_lsn,
-									  arguments->checksum_version,
-									  arguments->backup_version))
-					arguments->corrupted = true;
+				elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
+						file->path, file->crc, crc);
+				arguments->corrupted = true;
 			}
 		}
+		else
+		{
+			/*
+			 * validate relation block by block
+			 * check page headers, checksums (if enabled)
+			 * and compute checksum of the file
+			 */
+			if (!check_file_pages(file, arguments->stop_lsn,
+								  arguments->checksum_version,
+								  arguments->backup_version))
+				arguments->corrupted = true;
+		}
 	}
 
 	/* Data files validation is successful */