Merge branch 'REL_2_5' into REL_2_5-PBCKP-304

packaging/Makefile.test

@@ -130,10 +130,10 @@ build/test_suse: build/test_suse_15.1 build/test_suse_15.2
 	@echo Suse: done
 
 build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13
-	@echo Rhel 15.1: done
+	@echo Suse 15.1: done
 
 build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 build/test_suse_15.2_14
-	@echo Rhel 15.1: done
+	@echo Suse 15.2: done
 
 define test_suse
 	docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \

packaging/test/scripts/rpm.sh

@@ -77,6 +77,12 @@ if [ ${DISTRIB} == 'centos' ] && [ ${DISTRIB_VERSION} == '8' ]; then
     dnf -qy module disable postgresql
 fi
 
+# PGDG doesn't support install of PG-9.6 from repo package anymore
+if [ ${PG_VERSION} == '9.6' ] && [ ${DISTRIB_VERSION} == '7' ]; then
+    # ugly hack: use repo settings from PG10
+    sed -i 's/10/9.6/' /etc/yum.repos.d/pgdg-redhat-all.repo
+fi
+
 yum install -y postgresql${PG_TOG}-server.x86_64
 export PGDATA=/var/lib/pgsql/${PG_VERSION}/data
 

src/archive.c

@@ -1375,11 +1375,11 @@ get_wal_file(const char *filename, const char *from_fullpath,
 #ifdef HAVE_LIBZ
 		/* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */
 		if (IsXLogFileName(filename))
-			rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, &errmsg);
+			rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg);
 		if (rc == FILE_MISSING)
 #endif
 			/* ... failing that, use uncompressed */
-			rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg);
+			rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg);
 
 		/* When not in prefetch mode, try to use partial file */
 		if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename))
@@ -1389,13 +1389,13 @@ get_wal_file(const char *filename, const char *from_fullpath,
 #ifdef HAVE_LIBZ
 			/* '.gz.partial' goes first ... */
 			snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath);
-			rc = fio_send_file_gz(from_partial, to_fullpath, out, &errmsg);
+			rc = fio_send_file_gz(from_partial, out, &errmsg);
 			if (rc == FILE_MISSING)
 #endif
 			{
 				/* ... failing that, use '.partial' */
 				snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath);
-				rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg);
+				rc = fio_send_file(from_partial, out, false, NULL, &errmsg);
 			}
 
 			if (rc == SEND_OK)

src/backup.c

@@ -946,10 +946,21 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo)
 	 */
 #ifdef PGPRO_VERSION
 	if (!res)
+	{
 		/* It seems we connected to PostgreSQL (not Postgres Pro) */
-		elog(ERROR, "%s was built with Postgres Pro %s %s, "
+		if(strcmp(PGPRO_EDITION, "1C") != 0)
-					"but connection is made with PostgreSQL %s",
+		{
-			 PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
+			elog(ERROR, "%s was built with Postgres Pro %s %s, "
+						"but connection is made with PostgreSQL %s",
+				 PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
+		}
+		/* We have PostgresPro for 1C and connect to PostgreSQL or PostgresPro for 1C
+		 * Check the major version
+		*/
+		if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0)
+			elog(ERROR, "%s was built with PostgrePro %s %s, but connection is made with %s",
+				 PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
+	}
 	else
 	{
 		if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 &&

src/catalog.c

@@ -1069,6 +1069,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		char		linked[MAXPGPATH];
 		char		compress_alg_string[MAXPGPATH];
 		int64		write_size,
+					uncompressed_size,
 					mode,		/* bit length of mode_t depends on platforms */
 					is_datafile,
 					is_cfs,
@@ -1132,8 +1133,30 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
 			file->hdr_size = (int) hdr_size;
 
-		if (file->external_dir_num == 0)
+		if (get_control_value_int64(buf, "full_size", &uncompressed_size, false))
+			file->uncompressed_size = uncompressed_size;
+		else
+			file->uncompressed_size = write_size;
+		if (!file->is_datafile || file->is_cfs)
+			file->size = file->uncompressed_size;
+
+		if (file->external_dir_num == 0 && S_ISREG(file->mode))
+		{
+			bool is_datafile = file->is_datafile;
 			set_forkname(file);
+			if (is_datafile != file->is_datafile)
+			{
+				if (is_datafile)
+					elog(WARNING, "File '%s' was stored as datafile, but looks like it is not",
+						 file->rel_path);
+				else
+					elog(WARNING, "File '%s' was stored as non-datafile, but looks like it is",
+						 file->rel_path);
+				/* Lets fail in tests */
+				Assert(file->is_datafile == file->is_datafile);
+				file->is_datafile = is_datafile;
+			}
+		}
 
 		parray_append(files, file);
 	}
@@ -2561,6 +2584,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 					file->external_dir_num,
 					file->dbOid);
 
+		if (file->uncompressed_size != 0 &&
+				file->uncompressed_size != file->write_size)
+			len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"",
+						   file->uncompressed_size);
+
 		if (file->is_datafile)
 			len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
 

src/data.c

@@ -799,13 +799,17 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
 	 * and its mtime is less than parent backup start time ... */
 	if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) &&
 		(prev_file && file->exists_in_prev &&
+		 file->size == prev_file->size &&
 		 file->mtime <= parent_backup_time))
 	{
 		/*
 		 * file could be deleted under our feets.
 		 * But then backup_non_data_file_internal will handle it safely
 		 */
-		file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
+		if (file->forkName != cfm)
+			file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
+		else
+			file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true);
 
 		/* ...and checksum is the same... */
 		if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@@ -1330,7 +1334,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
 	if (already_exists)
 	{
 		/* compare checksums of already existing file and backup file */
-		pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
+		pg_crc32 file_crc;
+		if (tmp_file->forkName == cfm &&
+			    tmp_file->uncompressed_size > tmp_file->write_size)
+			file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false);
+		else
+			file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
 
 		if (file_crc == tmp_file->crc)
 		{
@@ -1387,10 +1396,12 @@ backup_non_data_file_internal(const char *from_fullpath,
 							const char *to_fullpath, pgFile *file,
 							bool missing_ok)
 {
-	FILE	*in = NULL;
 	FILE	*out = NULL;
-	ssize_t  read_len = 0;
+	char	*errmsg = NULL;
-	char	*buf = NULL;
+	int 	rc;
+	bool	cut_zero_tail;
+
+	cut_zero_tail = file->forkName == cfm;
 
 	INIT_FILE_CRC32(true, file->crc);
 
@@ -1412,107 +1423,44 @@ backup_non_data_file_internal(const char *from_fullpath,
 
 	/* backup remote file  */
 	if (fio_is_remote(FIO_DB_HOST))
-	{
+		rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg);
-		char *errmsg = NULL;
-		int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg);
-
-		/* handle errors */
-		if (rc == FILE_MISSING)
-		{
-			/* maybe deleted, it's not error in case of backup */
-			if (missing_ok)
-			{
-				elog(LOG, "File \"%s\" is not found", from_fullpath);
-				file->write_size = FILE_NOT_FOUND;
-				goto cleanup;
-			}
-			else
-				elog(ERROR, "File \"%s\" is not found", from_fullpath);
-		}
-		else if (rc == WRITE_FAILED)
-			elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
-		else if (rc != SEND_OK)
-		{
-			if (errmsg)
-				elog(ERROR, "%s", errmsg);
-			else
-				elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
-		}
-
-		pg_free(errmsg);
-	}
-	/* backup local file */
 	else
+		rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg);
+
+	/* handle errors */
+	if (rc == FILE_MISSING)
 	{
-		/* open source file for read */
+		/* maybe deleted, it's not error in case of backup */
-		in = fopen(from_fullpath, PG_BINARY_R);
+		if (missing_ok)
-		if (in == NULL)
 		{
-			/* maybe deleted, it's not error in case of backup */
+			elog(LOG, "File \"%s\" is not found", from_fullpath);
-			if (errno == ENOENT)
+			file->write_size = FILE_NOT_FOUND;
-			{
+			goto cleanup;
-				if (missing_ok)
-				{
-					elog(LOG, "File \"%s\" is not found", from_fullpath);
-					file->write_size = FILE_NOT_FOUND;
-					goto cleanup;
-				}
-				else
-					elog(ERROR, "File \"%s\" is not found", from_fullpath);
-			}
-
-			elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath,
-				 strerror(errno));
-		}
-
-		/* disable stdio buffering for local input/output files to avoid triple buffering */
-		setvbuf(in, NULL, _IONBF, BUFSIZ);
-		setvbuf(out, NULL, _IONBF, BUFSIZ);
-
-		/* allocate 64kB buffer */
-		buf = pgut_malloc(CHUNK_SIZE);
-
-		/* copy content and calc CRC */
-		for (;;)
-		{
-			read_len = fread(buf, 1, CHUNK_SIZE, in);
-
-			if (ferror(in))
-				elog(ERROR, "Cannot read from file \"%s\": %s",
-					 from_fullpath, strerror(errno));
-
-			if (read_len > 0)
-			{
-				if (fwrite(buf, 1, read_len, out) != read_len)
-					elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath,
-						 strerror(errno));
-
-				/* update CRC */
-				COMP_FILE_CRC32(true, file->crc, buf, read_len);
-				file->read_size += read_len;
-			}
-
-			if (feof(in))
-				break;
 		}
+		else
+			elog(ERROR, "File \"%s\" is not found", from_fullpath);
+	}
+	else if (rc == WRITE_FAILED)
+		elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
+	else if (rc != SEND_OK)
+	{
+		if (errmsg)
+			elog(ERROR, "%s", errmsg);
+		else
+			elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
 	}
 
-	file->write_size = (int64) file->read_size;
+	file->uncompressed_size = file->read_size;
-
-	if (file->write_size > 0)
-		file->uncompressed_size = file->write_size;
 
 cleanup:
+	if (errmsg != NULL)
+		pg_free(errmsg);
+
 	/* finish CRC calculation and store into pgFile */
 	FIN_FILE_CRC32(true, file->crc);
 
-	if (in && fclose(in))
-		elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno));
-
 	if (out && fclose(out))
 		elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno));
-
-	pg_free(buf);
 }
 
 /*

src/dir.c

@@ -262,137 +262,6 @@ delete_file:
 	}
 }
 
-/*
- * Read the local file to compute its CRC.
- * We cannot make decision about file decompression because
- * user may ask to backup already compressed files and we should be
- * obvious about it.
- */
-pg_crc32
-pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
-{
-	FILE	   *fp;
-	pg_crc32	crc = 0;
-	char	   *buf;
-	size_t		len = 0;
-
-	INIT_FILE_CRC32(use_crc32c, crc);
-
-	/* open file in binary read mode */
-	fp = fopen(file_path, PG_BINARY_R);
-	if (fp == NULL)
-	{
-		if (errno == ENOENT)
-		{
-			if (missing_ok)
-			{
-				FIN_FILE_CRC32(use_crc32c, crc);
-				return crc;
-			}
-		}
-
-		elog(ERROR, "Cannot open file \"%s\": %s",
-			file_path, strerror(errno));
-	}
-
-	/* disable stdio buffering */
-	setvbuf(fp, NULL, _IONBF, BUFSIZ);
-	buf = pgut_malloc(STDIO_BUFSIZE);
-
-	/* calc CRC of file */
-	for (;;)
-	{
-		if (interrupted)
-			elog(ERROR, "interrupted during CRC calculation");
-
-		len = fread(buf, 1, STDIO_BUFSIZE, fp);
-
-		if (ferror(fp))
-			elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
-
-		/* update CRC */
-		COMP_FILE_CRC32(use_crc32c, crc, buf, len);
-
-		if (feof(fp))
-			break;
-	}
-
-	FIN_FILE_CRC32(use_crc32c, crc);
-	fclose(fp);
-	pg_free(buf);
-
-	return crc;
-}
-
-/*
- * Read the local file to compute its CRC.
- * We cannot make decision about file decompression because
- * user may ask to backup already compressed files and we should be
- * obvious about it.
- */
-pg_crc32
-pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok)
-{
-	gzFile    fp;
-	pg_crc32  crc = 0;
-	int       len = 0;
-	int       err;
-	char	 *buf;
-
-	INIT_FILE_CRC32(use_crc32c, crc);
-
-	/* open file in binary read mode */
-	fp = gzopen(file_path, PG_BINARY_R);
-	if (fp == NULL)
-	{
-		if (errno == ENOENT)
-		{
-			if (missing_ok)
-			{
-				FIN_FILE_CRC32(use_crc32c, crc);
-				return crc;
-			}
-		}
-
-		elog(ERROR, "Cannot open file \"%s\": %s",
-			file_path, strerror(errno));
-	}
-
-	buf = pgut_malloc(STDIO_BUFSIZE);
-
-	/* calc CRC of file */
-	for (;;)
-	{
-		if (interrupted)
-			elog(ERROR, "interrupted during CRC calculation");
-
-		len = gzread(fp, buf, STDIO_BUFSIZE);
-
-		if (len <= 0)
-		{
-			/* we either run into eof or error */
-			if (gzeof(fp))
-				break;
-			else
-			{
-				const char *err_str = NULL;
-
-                err_str = gzerror(fp, &err);
-                elog(ERROR, "Cannot read from compressed file %s", err_str);
-			}
-		}
-
-		/* update CRC */
-		COMP_FILE_CRC32(use_crc32c, crc, buf, len);
-	}
-
-	FIN_FILE_CRC32(use_crc32c, crc);
-	gzclose(fp);
-	pg_free(buf);
-
-	return crc;
-}
-
 void
 pgFileFree(void *file)
 {
@@ -762,20 +631,6 @@ dir_check_file(pgFile *file, bool backup_logs)
 
 			if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */
 				return CHECK_FALSE;
-			else if (file->forkName != none)
-				return CHECK_TRUE;
-
-			/* Set is_datafile flag */
-			{
-				char suffix[MAXFNAMELEN];
-
-				/* check if file is datafile */
-				sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid),
-									&(file->segno), suffix);
-				Assert(sscanf_res > 0); /* since first char is digit */
-				if (sscanf_res == 1 || sscanf_res == 2)
-					file->is_datafile = true;
-			}
 		}
 	}
 
@@ -1812,7 +1667,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
 								 FIO_BACKUP_HOST);
 	file->crc = pgFileGetCRC(database_map_path, true, false);
 	file->write_size = file->size;
-	file->uncompressed_size = file->read_size;
+	file->uncompressed_size = file->size;
 
 	parray_append(backup_files_list, file);
 }
@@ -1920,34 +1775,74 @@ pfilearray_clear_locks(parray *file_list)
 	}
 }
 
+static inline bool
+is_forkname(char *name, size_t *pos, const char *forkname)
+{
+	size_t fnlen = strlen(forkname);
+	if (strncmp(name + *pos, forkname, fnlen) != 0)
+		return false;
+	*pos += fnlen;
+	return true;
+}
+
+#define OIDCHARS 10
+
 /* Set forkName if possible */
-void
+bool
 set_forkname(pgFile *file)
 {
-	int name_len = strlen(file->name);
+	size_t i = 0;
+	uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */
 
-	/* Auxiliary fork of the relfile */
+	/* pretend it is not relation file */
-	if (name_len > 3 && strcmp(file->name + name_len - 3, "_vm") == 0)
+	file->relOid = 0;
+	file->forkName = none;
+	file->is_datafile = false;
+
+	for (i = 0; isdigit(file->name[i]); i++)
+	{
+		if (i == 0 && file->name[i] == '0')
+			return false;
+		oid = oid * 10 + file->name[i] - '0';
+	}
+	if (i == 0 || i > OIDCHARS || oid > UINT32_MAX)
+		return false;
+
+	/* usual fork name */
+	/* /^\d+_(vm|fsm|init|ptrack)$/ */
+	if (is_forkname(file->name, &i, "_vm"))
 		file->forkName = vm;
-
+	else if (is_forkname(file->name, &i, "_fsm"))
-	else if (name_len > 4 && strcmp(file->name + name_len - 4, "_fsm") == 0)
 		file->forkName = fsm;
-
+	else if (is_forkname(file->name, &i, "_init"))
-	else if (name_len > 4 && strcmp(file->name + name_len - 4, ".cfm") == 0)
-		file->forkName = cfm;
-
-	else if (name_len > 5 && strcmp(file->name + name_len - 5, "_init") == 0)
 		file->forkName = init;
-
+	else if (is_forkname(file->name, &i, "_ptrack"))
-	else if (name_len > 7 && strcmp(file->name + name_len - 7, "_ptrack") == 0)
 		file->forkName = ptrack;
 
-	// extract relOid for certain forks
+	/* segment number */
+	/* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */
+	if (file->name[i] == '.' && isdigit(file->name[i+1]))
+	{
+		for (i++; isdigit(file->name[i]); i++)
+			;
+	}
 
-	if ((file->forkName == vm ||
+	/* CFS "fork name" */
-		 file->forkName == fsm ||
+	if (file->forkName == none &&
-		 file->forkName == init ||
+		is_forkname(file->name, &i, ".cfm"))
-		 file->forkName == cfm) &&
+	{
-		(sscanf(file->name, "%u*", &(file->relOid)) != 1))
+		/* /^\d+(\.\d+)?.cfm$/ */
-		file->relOid = 0;
+		file->forkName = cfm;
+	}
+
+	/* If there are excess characters, it is not relation file */
+	if (file->name[i] != 0)
+	{
+		file->forkName = none;
+		return false;
+	}
+
+	file->relOid = oid;
+	file->is_datafile = file->forkName == none;
+	return true;
 }

src/merge.c

@@ -1078,7 +1078,7 @@ merge_files(void *arg)
 					tmp_file->hdr_crc = file->hdr_crc;
 				}
 				else
-					tmp_file->uncompressed_size = tmp_file->write_size;
+					tmp_file->uncompressed_size = tmp_file->uncompressed_size;
 
 				/* Copy header metadata from old map into a new one */
 				tmp_file->n_headers = file->n_headers;

src/pg_probackup.h

@@ -345,7 +345,7 @@ typedef enum ShowFormat
 #define BYTES_INVALID		(-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */
 #define FILE_NOT_FOUND		(-2) /* file disappeared during backup */
 #define BLOCKNUM_INVALID	(-1)
-#define PROGRAM_VERSION	"2.5.8"
+#define PROGRAM_VERSION	"2.5.9"
 
 /* update when remote agent API or behaviour changes */
 #define AGENT_PROTOCOL_VERSION 20509
@@ -1082,6 +1082,7 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path);
 extern void pgFileFree(void *file);
 
 extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok);
+extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok);
 extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok);
 
 extern int pgFileMapComparePath(const void *f1, const void *f2);
@@ -1097,7 +1098,7 @@ extern int pgCompareString(const void *str1, const void *str2);
 extern int pgPrefixCompareString(const void *str1, const void *str2);
 extern int pgCompareOid(const void *f1, const void *f2);
 extern void pfilearray_clear_locks(parray *file_list);
-extern void set_forkname(pgFile *file);
+extern bool set_forkname(pgFile *file);
 
 /* in data.c */
 extern bool check_data_file(ConnectionArgs *arguments, pgFile *file,
@@ -1244,9 +1245,11 @@ extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pg
 	                      XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version,
 	                      bool use_pagemap, BlockNumber *err_blknum, char **errormsg);
 /* return codes for fio_send_pages */
-extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg);
+extern int fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg);
-extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
+extern int fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail,
 														pgFile *file, char **errormsg);
+extern int fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail,
+						 pgFile *file, char **errormsg);
 
 extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink,
 						 bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num);

src/utils/file.c

@@ -18,6 +18,10 @@ static __thread int fio_stdin = 0;
 static __thread int fio_stderr = 0;
 static char *async_errormsg = NULL;
 
+#define PAGE_ZEROSEARCH_COARSE_GRANULARITY 4096
+#define PAGE_ZEROSEARCH_FINE_GRANULARITY 64
+static const char zerobuf[PAGE_ZEROSEARCH_COARSE_GRANULARITY] = {0};
+
 fio_location MyLocation;
 
 typedef struct
@@ -1362,14 +1366,18 @@ fio_sync(char const* path, fio_location location)
 
 enum {
 	GET_CRC32_DECOMPRESS = 1,
-	GET_CRC32_MISSING_OK = 2
+	GET_CRC32_MISSING_OK = 2,
+	GET_CRC32_TRUNCATED  = 4
 };
 
 /* Get crc32 of file */
-pg_crc32
+static pg_crc32
-fio_get_crc32(const char *file_path, fio_location location,
+fio_get_crc32_ex(const char *file_path, fio_location location,
-			  bool decompress, bool missing_ok)
+			  bool decompress, bool missing_ok, bool truncated)
 {
+	if (decompress && truncated)
+		elog(ERROR, "Could not calculate CRC for compressed truncated file");
+
 	if (fio_is_remote(location))
 	{
 		fio_header hdr;
@@ -1384,6 +1392,8 @@ fio_get_crc32(const char *file_path, fio_location location,
 			hdr.arg = GET_CRC32_DECOMPRESS;
 		if (missing_ok)
 			hdr.arg |= GET_CRC32_MISSING_OK;
+		if (truncated)
+			hdr.arg |= GET_CRC32_TRUNCATED;
 
 		IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
 		IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len);
@@ -1395,11 +1405,27 @@ fio_get_crc32(const char *file_path, fio_location location,
 	{
 		if (decompress)
 			return pgFileGetCRCgz(file_path, true, missing_ok);
+		else if (truncated)
+			return pgFileGetCRCTruncated(file_path, true, missing_ok);
 		else
 			return pgFileGetCRC(file_path, true, missing_ok);
 	}
 }
 
+pg_crc32
+fio_get_crc32(const char *file_path, fio_location location,
+			  bool decompress, bool missing_ok)
+{
+	return fio_get_crc32_ex(file_path, location, decompress, missing_ok, false);
+}
+
+pg_crc32
+fio_get_crc32_truncated(const char *file_path, fio_location location,
+						bool missing_ok)
+{
+	return fio_get_crc32_ex(file_path, location, false, missing_ok, true);
+}
+
 /* Remove file */
 int
 fio_unlink(char const* path, fio_location location)
@@ -2460,7 +2486,7 @@ cleanup:
  *   REMOTE_ERROR (-6)
  */
 int
-fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg)
+fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg)
 {
 	fio_header hdr;
 	int exit_code = SEND_OK;
@@ -2609,6 +2635,105 @@ cleanup:
 	return exit_code;
 }
 
+typedef struct send_file_state {
+	bool		calc_crc;
+	uint32_t	crc;
+	int64_t		read_size;
+	int64_t		write_size;
+} send_file_state;
+
+/* find page border of all-zero tail */
+static size_t
+find_zero_tail(char *buf, size_t len)
+{
+	size_t i, l;
+	size_t granul = sizeof(zerobuf);
+
+	if (len == 0)
+		return 0;
+
+	/* fast check for last bytes */
+	l = Min(len, PAGE_ZEROSEARCH_FINE_GRANULARITY);
+	i = len - l;
+	if (memcmp(buf + i, zerobuf, l) != 0)
+		return len;
+
+	/* coarse search for zero tail */
+	i = (len-1) & ~(granul-1);
+	l = len - i;
+	for (;;)
+	{
+		if (memcmp(buf+i, zerobuf, l) != 0)
+		{
+			i += l;
+			break;
+		}
+		if (i == 0)
+			break;
+		i -= granul;
+		l = granul;
+	}
+
+	len = i;
+	/* search zero tail with finer granularity */
+	for (granul = sizeof(zerobuf)/2;
+		 len > 0 && granul >= PAGE_ZEROSEARCH_FINE_GRANULARITY;
+		 granul /= 2)
+	{
+		if (granul > l)
+			continue;
+		i = (len-1) & ~(granul-1);
+		l = len - i;
+		if (memcmp(buf+i, zerobuf, l) == 0)
+			len = i;
+	}
+
+	return len;
+}
+
+static void
+fio_send_file_crc(send_file_state* st, char *buf, size_t len)
+{
+	int64_t 	write_size;
+
+	if (!st->calc_crc)
+		return;
+
+	write_size = st->write_size;
+	while (st->read_size > write_size)
+	{
+		size_t	crc_len = Min(st->read_size - write_size, sizeof(zerobuf));
+		COMP_FILE_CRC32(true, st->crc, zerobuf, crc_len);
+		write_size += crc_len;
+	}
+
+	if (len > 0)
+		COMP_FILE_CRC32(true, st->crc, buf, len);
+}
+
+static bool
+fio_send_file_write(FILE* out, send_file_state* st, char *buf, size_t len)
+{
+	if (len == 0)
+		return true;
+
+	if (st->read_size > st->write_size &&
+		fseeko(out, st->read_size, SEEK_SET) != 0)
+	{
+		return false;
+	}
+
+	if (fwrite(buf, 1, len, out) != len)
+	{
+		return false;
+	}
+
+	st->read_size += len;
+	st->write_size = st->read_size;
+
+	return true;
+}
+
 /* Receive chunks of data and write them to destination file.
  * Return codes:
  *   SEND_OK       (0)
@@ -2621,13 +2746,22 @@ cleanup:
  * If pgFile is not NULL then we must calculate crc and read_size for it.
  */
 int
-fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
+fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail,
 												pgFile *file, char **errormsg)
 {
 	fio_header hdr;
 	int exit_code = SEND_OK;
 	size_t path_len = strlen(from_fullpath) + 1;
 	char *buf = pgut_malloc(CHUNK_SIZE);    /* buffer */
+	send_file_state st = {false, 0, 0, 0};
+
+	memset(&hdr, 0, sizeof(hdr));
+
+	if (file)
+	{
+		st.calc_crc = true;
+		st.crc = file->crc;
+	}
 
 	hdr.cop = FIO_SEND_FILE;
 	hdr.size = path_len;
@@ -2645,6 +2779,37 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
 
 		if (hdr.cop == FIO_SEND_FILE_EOF)
 		{
+			if (st.write_size < st.read_size)
+			{
+				if (!cut_zero_tail)
+				{
+					/*
+					 * We still need to calc crc for zero tail.
+					 */
+					fio_send_file_crc(&st, NULL, 0);
+
+					/*
+					 * Let's write single zero byte to the end of file to restore
+					 * logical size.
+					 * Well, it would be better to use ftruncate here actually,
+					 * but then we need to change interface.
+					 */
+					st.read_size -= 1;
+					buf[0] = 0;
+					if (!fio_send_file_write(out, &st, buf, 1))
+					{
+						exit_code = WRITE_FAILED;
+						break;
+					}
+				}
+			}
+
+			if (file)
+			{
+				file->crc = st.crc;
+				file->read_size = st.read_size;
+				file->write_size = st.write_size;
+			}
 			break;
 		}
 		else if (hdr.cop == FIO_ERROR)
@@ -2665,17 +2830,23 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
 			IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size);
 
 			/* We have received a chunk of data data, lets write it out */
-			if (fwrite(buf, 1, hdr.size, out) != hdr.size)
+			fio_send_file_crc(&st, buf, hdr.size);
+			if (!fio_send_file_write(out, &st, buf, hdr.size))
 			{
 				exit_code = WRITE_FAILED;
 				break;
 			}
+		}
+		else if (hdr.cop == FIO_PAGE_ZERO)
+		{
+			Assert(hdr.size == 0);
+			Assert(hdr.arg <= CHUNK_SIZE);
 
-			if (file)
+			/*
-			{
+			 * We have received a chunk of zero data, lets just think we
-				file->read_size += hdr.size;
+			 * wrote it.
-				COMP_FILE_CRC32(true, file->crc, buf, hdr.size);
+			 */
-			}
+			st.read_size += hdr.arg;
 		}
 		else
 		{
@@ -2691,6 +2862,128 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
 	return exit_code;
 }
 
+int
+fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail,
+			  pgFile *file, char **errormsg)
+{
+	FILE* in;
+	char* buf;
+	size_t read_len, non_zero_len;
+	int exit_code = SEND_OK;
+	send_file_state st = {false, 0, 0, 0};
+
+	if (file)
+	{
+		st.calc_crc = true;
+		st.crc = file->crc;
+	}
+
+	/* open source file for read */
+	in = fopen(from_fullpath, PG_BINARY_R);
+	if (in == NULL)
+	{
+		/* maybe deleted, it's not error in case of backup */
+		if (errno == ENOENT)
+			return FILE_MISSING;
+
+
+		*errormsg = psprintf("Cannot open file \"%s\": %s", from_fullpath,
+			 strerror(errno));
+		return OPEN_FAILED;
+	}
+
+	/* disable stdio buffering for local input/output files to avoid triple buffering */
+	setvbuf(in, NULL, _IONBF, BUFSIZ);
+	setvbuf(out, NULL, _IONBF, BUFSIZ);
+
+	/* allocate 64kB buffer */
+	buf = pgut_malloc(CHUNK_SIZE);
+
+	/* copy content and calc CRC */
+	for (;;)
+	{
+		read_len = fread(buf, 1, CHUNK_SIZE, in);
+
+		if (ferror(in))
+		{
+			*errormsg = psprintf("Cannot read from file \"%s\": %s",
+								 from_fullpath, strerror(errno));
+			exit_code = READ_FAILED;
+			goto cleanup;
+		}
+
+		if (read_len > 0)
+		{
+			non_zero_len = find_zero_tail(buf, read_len);
+			/*
+			 * It is dirty trick to silence warnings in CFS GC process:
+			 * backup at least cfs header size bytes.
+			 */
+			if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
+				st.read_size + read_len > 0)
+			{
+				non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
+								   st.read_size + read_len);
+				non_zero_len -= st.read_size;
+			}
+			if (non_zero_len > 0)
+			{
+				fio_send_file_crc(&st, buf, non_zero_len);
+				if (!fio_send_file_write(out, &st, buf, non_zero_len))
+				{
+					exit_code = WRITE_FAILED;
+					goto cleanup;
+				}
+			}
+			if (non_zero_len < read_len)
+			{
+				/* Just pretend we wrote it. */
+				st.read_size += read_len - non_zero_len;
+			}
+		}
+
+		if (feof(in))
+			break;
+	}
+
+	if (st.write_size < st.read_size)
+	{
+		if (!cut_zero_tail)
+		{
+			/*
+			 * We still need to calc crc for zero tail.
+			 */
+			fio_send_file_crc(&st, NULL, 0);
+
+			/*
+			 * Let's write single zero byte to the end of file to restore
+			 * logical size.
+			 * Well, it would be better to use ftruncate here actually,
+			 * but then we need to change interface.
+			 */
+			st.read_size -= 1;
+			buf[0] = 0;
+			if (!fio_send_file_write(out, &st, buf, 1))
+			{
+				exit_code = WRITE_FAILED;
+				goto cleanup;
+			}
+		}
+	}
+
+	if (file)
+	{
+		file->crc = st.crc;
+		file->read_size = st.read_size;
+		file->write_size = st.write_size;
+	}
+
+cleanup:
+	free(buf);
+	fclose(in);
+	return exit_code;
+}
+
 /* Send file content
  * On error we return FIO_ERROR message with following codes
  *  FIO_ERROR:
@@ -2709,6 +3002,7 @@ fio_send_file_impl(int out, char const* path)
 	fio_header hdr;
 	char      *buf = pgut_malloc(CHUNK_SIZE);
 	size_t	   read_len = 0;
+	int64_t	   read_size = 0;
 	char      *errormsg = NULL;
 
 	/* open source file for read */
@@ -2751,6 +3045,7 @@ fio_send_file_impl(int out, char const* path)
 	for (;;)
 	{
 		read_len = fread(buf, 1, CHUNK_SIZE, fp);
+		memset(&hdr, 0, sizeof(hdr));
 
 		/* report error */
 		if (ferror(fp))
@@ -2771,10 +3066,36 @@ fio_send_file_impl(int out, char const* path)
 		if (read_len > 0)
 		{
 			/* send chunk */
-			hdr.cop = FIO_PAGE;
+			int64_t non_zero_len = find_zero_tail(buf, read_len);
-			hdr.size = read_len;
+			/*
-			IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
+			 * It is dirty trick to silence warnings in CFS GC process:
-			IO_CHECK(fio_write_all(out, buf, read_len), read_len);
+			 * backup at least cfs header size bytes.
+			 */
+			if (read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
+				read_size + read_len > 0)
+			{
+				non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
+								   read_size + read_len);
+				non_zero_len -= read_size;
+			}
+
+			if (non_zero_len > 0)
+			{
+				hdr.cop = FIO_PAGE;
+				hdr.size = non_zero_len;
+				IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
+				IO_CHECK(fio_write_all(out, buf, non_zero_len), non_zero_len);
+			}
+
+			if (non_zero_len < read_len)
+			{
+				hdr.cop = FIO_PAGE_ZERO;
+				hdr.size = 0;
+				hdr.arg = read_len - non_zero_len;
+				IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
+			}
+
+			read_size += read_len;
 		}
 
 		if (feof(fp))
@@ -2793,6 +3114,210 @@ cleanup:
 	return;
 }
 
+/*
+ * Read the local file to compute its CRC.
+ * We cannot make decision about file decompression because
+ * user may ask to backup already compressed files and we should be
+ * obvious about it.
+ */
+pg_crc32
+pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
+{
+	FILE	   *fp;
+	pg_crc32	crc = 0;
+	char	   *buf;
+	size_t		len = 0;
+
+	INIT_FILE_CRC32(use_crc32c, crc);
+
+	/* open file in binary read mode */
+	fp = fopen(file_path, PG_BINARY_R);
+	if (fp == NULL)
+	{
+		if (errno == ENOENT)
+		{
+			if (missing_ok)
+			{
+				FIN_FILE_CRC32(use_crc32c, crc);
+				return crc;
+			}
+		}
+
+		elog(ERROR, "Cannot open file \"%s\": %s",
+			 file_path, strerror(errno));
+	}
+
+	/* disable stdio buffering */
+	setvbuf(fp, NULL, _IONBF, BUFSIZ);
+	buf = pgut_malloc(STDIO_BUFSIZE);
+
+	/* calc CRC of file */
+	for (;;)
+	{
+		if (interrupted)
+			elog(ERROR, "interrupted during CRC calculation");
+
+		len = fread(buf, 1, STDIO_BUFSIZE, fp);
+
+		if (ferror(fp))
+			elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
+
+		/* update CRC */
+		COMP_FILE_CRC32(use_crc32c, crc, buf, len);
+
+		if (feof(fp))
+			break;
+	}
+
+	FIN_FILE_CRC32(use_crc32c, crc);
+	fclose(fp);
+	pg_free(buf);
+
+	return crc;
+}
+
+/*
+ * Read the local file to compute CRC for it extened to real_size.
+ */
+pg_crc32
+pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok)
+{
+	FILE	   *fp;
+	char	   *buf;
+	size_t		len = 0;
+	size_t		non_zero_len;
+	send_file_state st = {true, 0, 0, 0};
+
+	INIT_FILE_CRC32(use_crc32c, st.crc);
+
+	/* open file in binary read mode */
+	fp = fopen(file_path, PG_BINARY_R);
+	if (fp == NULL)
+	{
+		if (errno == ENOENT)
+		{
+			if (missing_ok)
+			{
+				FIN_FILE_CRC32(use_crc32c, st.crc);
+				return st.crc;
+			}
+		}
+
+		elog(ERROR, "Cannot open file \"%s\": %s",
+			 file_path, strerror(errno));
+	}
+
+	/* disable stdio buffering */
+	setvbuf(fp, NULL, _IONBF, BUFSIZ);
+	buf = pgut_malloc(CHUNK_SIZE);
+
+	/* calc CRC of file */
+	for (;;)
+	{
+		if (interrupted)
+			elog(ERROR, "interrupted during CRC calculation");
+
+		len = fread(buf, 1, STDIO_BUFSIZE, fp);
+
+		if (ferror(fp))
+			elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
+
+		non_zero_len = find_zero_tail(buf, len);
+		/* same trick as in fio_send_file */
+		if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
+			st.read_size + len > 0)
+		{
+			non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
+							   st.read_size + len);
+			non_zero_len -= st.read_size;
+		}
+		if (non_zero_len)
+		{
+			fio_send_file_crc(&st, buf, non_zero_len);
+			st.write_size += st.read_size + non_zero_len;
+		}
+		st.read_size += len;
+
+		if (feof(fp))
+			break;
+	}
+
+	FIN_FILE_CRC32(use_crc32c, st.crc);
+	fclose(fp);
+	pg_free(buf);
+
+	return st.crc;
+}
+
+/*
+ * Read the local file to compute its CRC.
+ * We cannot make decision about file decompression because
+ * user may ask to backup already compressed files and we should be
+ * obvious about it.
+ */
+pg_crc32
+pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok)
+{
+	gzFile    fp;
+	pg_crc32  crc = 0;
+	int       len = 0;
+	int       err;
+	char	 *buf;
+
+	INIT_FILE_CRC32(use_crc32c, crc);
+
+	/* open file in binary read mode */
+	fp = gzopen(file_path, PG_BINARY_R);
+	if (fp == NULL)
+	{
+		if (errno == ENOENT)
+		{
+			if (missing_ok)
+			{
+				FIN_FILE_CRC32(use_crc32c, crc);
+				return crc;
+			}
+		}
+
+		elog(ERROR, "Cannot open file \"%s\": %s",
+			 file_path, strerror(errno));
+	}
+
+	buf = pgut_malloc(STDIO_BUFSIZE);
+
+	/* calc CRC of file */
+	for (;;)
+	{
+		if (interrupted)
+			elog(ERROR, "interrupted during CRC calculation");
+
+		len = gzread(fp, buf, STDIO_BUFSIZE);
+
+		if (len <= 0)
+		{
+			/* we either run into eof or error */
+			if (gzeof(fp))
+				break;
+			else
+			{
+				const char *err_str = NULL;
+
+				err_str = gzerror(fp, &err);
+				elog(ERROR, "Cannot read from compressed file %s", err_str);
+			}
+		}
+
+		/* update CRC */
+		COMP_FILE_CRC32(use_crc32c, crc, buf, len);
+	}
+
+	FIN_FILE_CRC32(use_crc32c, crc);
+	gzclose(fp);
+	pg_free(buf);
+
+	return crc;
+}
+
 /* Compile the array of files located on remote machine in directory root */
 static void
 fio_list_dir_internal(parray *files, const char *root, bool exclude,
@@ -3399,9 +3924,13 @@ fio_communicate(int in, int out)
 			IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
 			break;
 		  case FIO_GET_CRC32:
+			Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 ||
+				   (hdr.arg & (GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS)) == GET_CRC32_TRUNCATED);
 			/* calculate crc32 for a file */
 			if ((hdr.arg & GET_CRC32_DECOMPRESS))
 				crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
+			else if ((hdr.arg & GET_CRC32_TRUNCATED))
+				crc = pgFileGetCRCTruncated(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
 			else
 				crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
 			IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));

src/utils/file.h

@@ -56,7 +56,8 @@ typedef enum
 	FIO_CHECK_POSTMASTER,
 	FIO_GET_ASYNC_ERROR,
 	FIO_WRITE_ASYNC,
-	FIO_READLINK
+	FIO_READLINK,
+	FIO_PAGE_ZERO
 } fio_operations;
 
 typedef enum
@@ -122,6 +123,8 @@ extern void    fio_disconnect(void);
 extern int     fio_sync(char const* path, fio_location location);
 extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location,
 							  bool decompress, bool missing_ok);
+extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location,
+										bool missing_ok);
 
 extern int     fio_rename(char const* old_path, char const* new_path, fio_location location);
 extern int     fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location);

tests/backup.py

@@ -3251,10 +3251,15 @@ class BackupTest(ProbackupTest, unittest.TestCase):
             self.assertIn(
                 'WARNING: backup in progress, stop backup',
                 log_content)
-            
+
-            self.assertIn(
+            if self.get_version(node) < 150000:
-                'FROM pg_catalog.pg_backup_stop',
+                self.assertIn(
-                log_content)
+                    'FROM pg_catalog.pg_stop_backup',
+                    log_content)
+            else:
+                self.assertIn(
+                    'FROM pg_catalog.pg_backup_stop',
+                    log_content)
             
             self.assertIn(
                 'setting its status to ERROR',

tests/cfs_backup.py

@@ -169,12 +169,18 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
             "ERROR: File pg_compression not found in {0}".format(
                 os.path.join(self.backup_dir, 'node', backup_id))
         )
-        self.assertTrue(
+
-            find_by_extensions(
+        # check cfm size
-                [os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
+        cfms = find_by_extensions(
-                ['.cfm']),
+            [os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
-            "ERROR: .cfm files not found in backup dir"
+            ['.cfm'])
-        )
+        self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir")
+        for cfm in cfms:
+            size = os.stat(cfm).st_size
+            self.assertLessEqual(size, 4096,
+                            "ERROR: {0} is not truncated (has size {1} > 4096)".format(
+                                cfm, size
+                            ))
 
     # @unittest.expectedFailure
     # @unittest.skip("skip")
@@ -411,6 +417,69 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
             "ERROR: .cfm files not found in backup dir"
         )
 
+    @unittest.skipUnless(ProbackupTest.enterprise, 'skip')
+    def test_page_doesnt_store_unchanged_cfm(self):
+        """
+        Case: Test page backup doesn't store cfm file if table were not modified
+        """
+
+        self.node.safe_psql(
+            "postgres",
+            "CREATE TABLE {0} TABLESPACE {1} "
+            "AS SELECT i AS id, MD5(i::text) AS text, "
+            "MD5(repeat(i::text,10))::tsvector AS tsvector "
+            "FROM generate_series(0,256) i".format('t1', tblspace_name)
+        )
+
+        try:
+            backup_id_full = self.backup_node(
+                self.backup_dir, 'node', self.node, backup_type='full')
+        except ProbackupException as e:
+            self.fail(
+                "ERROR: Full backup failed.\n {0} \n {1}".format(
+                    repr(self.cmd),
+                    repr(e.message)
+                )
+            )
+
+        self.assertTrue(
+            find_by_extensions(
+                [os.path.join(self.backup_dir, 'backups', 'node', backup_id_full)],
+                ['.cfm']),
+            "ERROR: .cfm files not found in backup dir"
+        )
+
+        try:
+            backup_id = self.backup_node(
+                self.backup_dir, 'node', self.node, backup_type='page')
+        except ProbackupException as e:
+            self.fail(
+                "ERROR: Incremental backup failed.\n {0} \n {1}".format(
+                    repr(self.cmd),
+                    repr(e.message)
+                )
+            )
+
+        show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
+        self.assertEqual(
+            "OK",
+            show_backup["status"],
+            "ERROR: Incremental backup status is not valid. \n "
+            "Current backup status={0}".format(show_backup["status"])
+        )
+        self.assertTrue(
+            find_by_name(
+                [self.get_tblspace_path(self.node, tblspace_name)],
+                ['pg_compression']),
+            "ERROR: File pg_compression not found"
+        )
+        self.assertFalse(
+            find_by_extensions(
+                [os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
+                ['.cfm']),
+            "ERROR: .cfm files is found in backup dir"
+        )
+
     # @unittest.expectedFailure
     # @unittest.skip("skip")
     @unittest.skipUnless(ProbackupTest.enterprise, 'skip')

tests/cfs_catchup.py

@@ -0,0 +1,127 @@
+import os
+import unittest
+import random
+import shutil
+
+from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file
+from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
+
+module_name = 'cfs_catchup'
+tblspace_name = 'cfs_tblspace'
+
+
+class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase):
+    def setUp(self):
+        self.fname = self.id().split('.')[3]
+
+    @unittest.skipUnless(ProbackupTest.enterprise, 'skip')
+    def test_full_catchup_with_tablespace(self):
+        """
+        Test tablespace transfers
+        """
+        # preparation
+        src_pg = self.make_simple_node(
+            base_dir = os.path.join(module_name, self.fname, 'src'),
+            set_replication = True
+        )
+        src_pg.slow_start()
+        tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old')
+        self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path, cfs=True)
+        src_pg.safe_psql(
+            "postgres",
+            "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer")
+        src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
+        src_pg.safe_psql(
+            "postgres",
+            "CHECKPOINT")
+
+        # do full catchup with tablespace mapping
+        dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
+        tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new')
+        self.catchup_node(
+            backup_mode = 'FULL',
+            source_pgdata = src_pg.data_dir,
+            destination_node = dst_pg,
+            options = [
+                '-d', 'postgres',
+                '-p', str(src_pg.port),
+                '--stream',
+                '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path)
+            ]
+        )
+
+        # 1st check: compare data directories
+        self.compare_pgdata(
+            self.pgdata_content(src_pg.data_dir),
+            self.pgdata_content(dst_pg.data_dir)
+        )
+
+        # check cfm size
+        cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm'])
+        self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir")
+        for cfm in cfms:
+            size = os.stat(cfm).st_size
+            self.assertLessEqual(size, 4096,
+                                 "ERROR: {0} is not truncated (has size {1} > 4096)".format(
+                                     cfm, size
+                                 ))
+
+        # make changes in master tablespace
+        src_pg.safe_psql(
+            "postgres",
+            "UPDATE ultimate_question SET answer = -1")
+        src_pg.safe_psql(
+            "postgres",
+            "CHECKPOINT")
+
+        # run&recover catchup'ed instance
+        dst_options = {}
+        dst_options['port'] = str(dst_pg.port)
+        self.set_auto_conf(dst_pg, dst_options)
+        dst_pg.slow_start()
+
+        # 2nd check: run verification query
+        dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
+        self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy')
+
+        # and now delta backup
+        dst_pg.stop()
+
+        self.catchup_node(
+            backup_mode = 'DELTA',
+            source_pgdata = src_pg.data_dir,
+            destination_node = dst_pg,
+            options = [
+                '-d', 'postgres',
+                '-p', str(src_pg.port),
+                '--stream',
+                '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path)
+            ]
+        )
+
+        # check cfm size again
+        cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm'])
+        self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir")
+        for cfm in cfms:
+            size = os.stat(cfm).st_size
+            self.assertLessEqual(size, 4096,
+                                 "ERROR: {0} is not truncated (has size {1} > 4096)".format(
+                                     cfm, size
+                                 ))
+
+        # run&recover catchup'ed instance
+        dst_options = {}
+        dst_options['port'] = str(dst_pg.port)
+        self.set_auto_conf(dst_pg, dst_options)
+        dst_pg.slow_start()
+
+
+        # 3rd check: run verification query
+        src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
+        dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
+        self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy')
+
+        # Cleanup
+        src_pg.stop()
+        dst_pg.stop()
+        self.del_test_dir(module_name, self.fname)

tests/expected/option_version.out

@@ -1 +1 @@
-pg_probackup 2.5.8
+pg_probackup 2.5.9

tests/helpers/ptrack_helpers.py

@@ -1751,8 +1751,18 @@ class ProbackupTest(object):
                 file_relpath = os.path.relpath(file_fullpath, pgdata)
                 directory_dict['files'][file_relpath] = {'is_datafile': False}
                 with open(file_fullpath, 'rb') as f:
-                    directory_dict['files'][file_relpath]['md5'] = hashlib.md5(f.read()).hexdigest()
+                    content = f.read()
-                    f.close()
+                    # truncate cfm's content's zero tail
+                    if file_relpath.endswith('.cfm'):
+                        zero64 = b"\x00"*64
+                        l = len(content)
+                        while l > 64:
+                            s = (l - 1) & ~63
+                            if content[s:l] != zero64[:l-s]:
+                                break
+                            l = s
+                        content = content[:l]
+                    directory_dict['files'][file_relpath]['md5'] = hashlib.md5(content).hexdigest()
 #                directory_dict['files'][file_relpath]['md5'] = hashlib.md5(
 #                    f = open(file_fullpath, 'rb').read()).hexdigest()