Merge branch 'master' of https://git.postgrespro.ru/pgpro-dev/pg_probackup into cfs

Conflicts:
	tests/__init__.py

src/data.c

@@ -22,6 +22,7 @@
 #include <common/pg_lzcompress.h>
 #include <zlib.h>
 
+/* Implementation of zlib compression method */
 static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
 {
 	uLongf compressed_size = dst_size;
@@ -29,6 +30,7 @@ static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t
 	return rc == Z_OK ? compressed_size : rc;
 }
 
+/* Implementation of zlib compression method */
 static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
 {
 	uLongf dest_len = dst_size;
@@ -36,6 +38,10 @@ static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_
 	return rc == Z_OK ? dest_len : rc;
 }
 
+/*
+ * Compresses source into dest using algorithm. Returns the number of bytes
+ * written in the destination buffer, or -1 if compression fails.
+ */
 static size_t
 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg)
 {
@@ -53,6 +59,10 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre
 	return -1;
 }
 
+/*
+ * Decompresses source into dest using algorithm. Returns the number of bytes
+ * decompressed in the destination buffer, or -1 if decompression fails.
+ */
 static size_t
 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg)
 {
@@ -70,8 +80,10 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, Comp
 	return -1;
 }
 
-
-
+/*
+ * When copying datafiles to backup we validate and compress them block
+ * by block. Thus special header is required for each data block.
+ */
 typedef struct BackupPageHeader
 {
 	BlockNumber	block;			/* block number */
@@ -125,6 +137,11 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
 	header.block = blknum;
 	offset = blknum * BLCKSZ;
 
+	/*
+	 * Read the page and verify its header and checksum.
+	 * Under high write load it's possible that we've read partly
+	 * flushed page, so try several times befor throwing an error.
+	 */
 	while(try_checksum--)
 	{
 		if (fseek(in, offset, SEEK_SET) != 0)
@@ -176,14 +193,14 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
 				elog(ERROR, "File: %s blknum %u have wrong page header.", file->path, blknum);
 		}
 
-		/* If the page hasn't changed since previous backup, don't backup it. */
-		if (!XLogRecPtrIsInvalid(prev_backup_start_lsn)
-			&& !XLogRecPtrIsInvalid(page_lsn)
-			&& page_lsn < prev_backup_start_lsn)
-		{
-			*n_skipped += 1;
-			return;
-		}
+// 		/* If the page hasn't changed since previous backup, don't backup it. */
+// 		if (!XLogRecPtrIsInvalid(prev_backup_start_lsn)
+// 			&& !XLogRecPtrIsInvalid(page_lsn)
+// 			&& page_lsn < prev_backup_start_lsn)
+// 		{
+// 			*n_skipped += 1;
+// 			return;
+// 		}
 
 		/* Verify checksum */
 		if(current.checksum_version)
@@ -223,12 +240,14 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
 	Assert (header.compressed_size <= BLCKSZ);
 	write_buffer_size = sizeof(header);
 
+	/* The page was successfully compressed */
 	if (header.compressed_size > 0)
 	{
 		memcpy(write_buffer, &header, sizeof(header));
 		memcpy(write_buffer + sizeof(header), compressed_page.data, header.compressed_size);
 		write_buffer_size += MAXALIGN(header.compressed_size);
 	}
+	/* The page compression failed. Write it as is. */
 	else
 	{
 		header.compressed_size = BLCKSZ;
@@ -324,8 +343,7 @@ backup_data_file(const char *from_root, const char *to_root,
 
 	/*
 	 * Read each page, verify checksum and write it to backup.
-	 * If page map is not empty we scan only changed blocks, otherwise
-	 * backup all pages of the relation.
+	 * If page map is empty backup all pages of the relation.
 	 */
 	if (file->pagemap.bitmapsize == 0)
 	{
@@ -336,6 +354,7 @@ backup_data_file(const char *from_root, const char *to_root,
 			n_blocks_read++;
 		}
 	}
+	/* If page map is not empty we scan only changed blocks, */
 	else
 	{
 		datapagemap_iterator_t *iter;
@@ -371,7 +390,7 @@ backup_data_file(const char *from_root, const char *to_root,
 	FIN_CRC32C(file->crc);
 
 	/*
-	 * If we have pagemap then file can't be a zero size.
+	 * If we have pagemap then file in the backup can't be a zero size.
 	 * Otherwise, we will clear the last file.
 	 */
 	if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped)
@@ -385,117 +404,6 @@ backup_data_file(const char *from_root, const char *to_root,
 	return true;
 }
 
-/*
- * Restore compressed file that was backed up partly.
- */
-static void
-restore_file_partly(const char *from_root,const char *to_root, pgFile *file)
-{
-	FILE	   *in;
-	FILE	   *out;
-	size_t		read_len = 0;
-	int			errno_tmp;
-	struct stat	st;
-	char		to_path[MAXPGPATH];
-	char		buf[BLCKSZ];
-	size_t write_size = 0;
-
-	join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
-	/* open backup mode file for read */
-	in = fopen(file->path, "r");
-	if (in == NULL)
-	{
-		elog(ERROR, "cannot open backup file \"%s\": %s", file->path,
-			strerror(errno));
-	}
-	out = fopen(to_path, "r+");
-
-	/* stat source file to change mode of destination file */
-	if (fstat(fileno(in), &st) == -1)
-	{
-		fclose(in);
-		fclose(out);
-		elog(ERROR, "cannot stat \"%s\": %s", file->path,
-			 strerror(errno));
-	}
-
-	if (fseek(out, 0, SEEK_END) < 0)
-		elog(ERROR, "cannot seek END of \"%s\": %s",
-				to_path, strerror(errno));
-
-	/* copy everything from backup to the end of the file */
-	for (;;)
-	{
-		if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf))
-			break;
-
-		if (fwrite(buf, 1, read_len, out) != read_len)
-		{
-			errno_tmp = errno;
-			/* oops */
-			fclose(in);
-			fclose(out);
-			elog(ERROR, "cannot write to \"%s\": %s", to_path,
-				 strerror(errno_tmp));
-		}
-		write_size += read_len;
-	}
-
-	errno_tmp = errno;
-	if (!feof(in))
-	{
-		fclose(in);
-		fclose(out);
-		elog(ERROR, "cannot read backup mode file \"%s\": %s",
-			 file->path, strerror(errno_tmp));
-	}
-
-	/* copy odd part. */
-	if (read_len > 0)
-	{
-		if (fwrite(buf, 1, read_len, out) != read_len)
-		{
-			errno_tmp = errno;
-			/* oops */
-			fclose(in);
-			fclose(out);
-			elog(ERROR, "cannot write to \"%s\": %s", to_path,
-				 strerror(errno_tmp));
-		}
-
-		write_size += read_len;
-	}
-
-
-	/* update file permission */
-	if (chmod(to_path, file->mode) == -1)
-	{
-		int errno_tmp = errno;
-
-		fclose(in);
-		fclose(out);
-		elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
-			strerror(errno_tmp));
-	}
-
-	if (fflush(out) != 0 ||
-		fsync(fileno(out)) != 0 ||
-		fclose(out))
-		elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno));
-	fclose(in);
-}
-
-void
-restore_compressed_file(const char *from_root,
-						const char *to_root,
-						pgFile *file)
-{
-	if (!file->is_partial_copy)
-		copy_file(from_root, to_root, file);
-	else
-		restore_file_partly(from_root, to_root, file);
-}
-
 /*
  * Restore files in the from_root directory to the to_root directory with
  * same relative path.
@@ -522,7 +430,7 @@ restore_data_file(const char *from_root,
 
 	/*
 	 * Open backup file for write. 	We use "r+" at first to overwrite only
-	 * modified pages for differential restore. If the file is not exists,
+	 * modified pages for differential restore. If the file does not exist,
 	 * re-open it with "w" to create an empty file.
 	 */
 	join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
@@ -856,136 +764,6 @@ copy_wal_file(const char *from_path, const char *to_path)
 	fclose(in);
 }
 
-/*
- * Save part of the file into backup.
- * skip_size - size of the file in previous backup. We can skip it
- *			   and copy just remaining part of the file
- */
-bool
-copy_file_partly(const char *from_root, const char *to_root,
-				 pgFile *file, size_t skip_size)
-{
-	char		to_path[MAXPGPATH];
-	FILE	   *in;
-	FILE	   *out;
-	size_t		read_len = 0;
-	int			errno_tmp;
-	struct stat	st;
-	char		buf[BLCKSZ];
-
-	/* reset size summary */
-	file->read_size = 0;
-	file->write_size = 0;
-
-	/* open backup mode file for read */
-	in = fopen(file->path, "r");
-	if (in == NULL)
-	{
-		/* maybe deleted, it's not error */
-		if (errno == ENOENT)
-			return false;
-
-		elog(ERROR, "cannot open source file \"%s\": %s", file->path,
-			 strerror(errno));
-	}
-
-	/* open backup file for write  */
-	join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
-
-	out = fopen(to_path, "w");
-	if (out == NULL)
-	{
-		int errno_tmp = errno;
-		fclose(in);
-		elog(ERROR, "cannot open destination file \"%s\": %s",
-			 to_path, strerror(errno_tmp));
-	}
-
-	/* stat source file to change mode of destination file */
-	if (fstat(fileno(in), &st) == -1)
-	{
-		fclose(in);
-		fclose(out);
-		elog(ERROR, "cannot stat \"%s\": %s", file->path,
-			 strerror(errno));
-	}
-
-	if (fseek(in, skip_size, SEEK_SET) < 0)
-		elog(ERROR, "cannot seek %lu of \"%s\": %s",
-				skip_size, file->path, strerror(errno));
-
-	/*
-	 * copy content
-	 * NOTE: Now CRC is not computed for compressed files now.
-	 */
-	for (;;)
-	{
-		if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf))
-			break;
-
-		if (fwrite(buf, 1, read_len, out) != read_len)
-		{
-			errno_tmp = errno;
-			/* oops */
-			fclose(in);
-			fclose(out);
-			elog(ERROR, "cannot write to \"%s\": %s", to_path,
-				 strerror(errno_tmp));
-		}
-
-		file->write_size += sizeof(buf);
-		file->read_size += sizeof(buf);
-	}
-
-	errno_tmp = errno;
-	if (!feof(in))
-	{
-		fclose(in);
-		fclose(out);
-		elog(ERROR, "cannot read backup mode file \"%s\": %s",
-			 file->path, strerror(errno_tmp));
-	}
-
-	/* copy odd part. */
-	if (read_len > 0)
-	{
-		if (fwrite(buf, 1, read_len, out) != read_len)
-		{
-			errno_tmp = errno;
-			/* oops */
-			fclose(in);
-			fclose(out);
-			elog(ERROR, "cannot write to \"%s\": %s", to_path,
-				 strerror(errno_tmp));
-		}
-
-		file->write_size += read_len;
-		file->read_size += read_len;
-	}
-
-	/* update file permission */
-	if (chmod(to_path, st.st_mode) == -1)
-	{
-		errno_tmp = errno;
-		fclose(in);
-		fclose(out);
-		elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
-			 strerror(errno_tmp));
-	}
-
-	/* add meta information needed for recovery */
-	file->is_partial_copy = true;
-
-	if (fflush(out) != 0 ||
-		fsync(fileno(out)) != 0 ||
-		fclose(out))
-		elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno));
-	fclose(in);
-
-	return true;
-}
-
-
 /*
  * Calculate checksum of various files which are not copied from PGDATA,
  * but created in process of backup, such as stream XLOG files,

src/dir.c

@@ -150,13 +150,15 @@ pgFileInit(const char *path)
 	file->linked = NULL;
 	file->pagemap.bitmap = NULL;
 	file->pagemap.bitmapsize = 0;
-	file->ptrack_path = NULL;
+	file->tblspcOid = 0;
+	file->dbOid = 0;
+	file->relOid = 0;
 	file->segno = 0;
+	file->is_database = false;
+	file->forkName = pgut_malloc(MAXPGPATH);
 	file->path = pgut_malloc(strlen(path) + 1);
 	strcpy(file->path, path);		/* enough buffer size guaranteed */
 	file->is_cfs = false;
-	file->generation = 0;
-	file->is_partial_copy = false;
 	file->compress_alg = NOT_DEFINED_COMPRESS;
 	return file;
 }
@@ -241,9 +243,11 @@ pgFileFree(void *file)
 
 	if (file_ptr->linked)
 		free(file_ptr->linked);
+
+	if (file_ptr->forkName)
+		free(file_ptr->forkName);
+
 	free(file_ptr->path);
-	if (file_ptr->ptrack_path != NULL)
-		free(file_ptr->ptrack_path);
 	free(file);
 }
 
@@ -378,6 +382,7 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
 	if (add_root)
 	{
 		/* Skip files */
+		/* TODO Consider moving this check to parse_backup_filelist_filenames */
 		if (!S_ISDIR(file->mode) && exclude)
 		{
 			char	    *file_name;
@@ -683,12 +688,6 @@ print_file_list(FILE *out, const parray *files, const char *root)
 		if (S_ISLNK(file->mode))
 			fprintf(out, ",\"linked\":\"%s\"", file->linked);
 
-#ifdef PGPRO_EE
-		if (file->is_cfs)
-			fprintf(out, ",\"is_cfs\":\"%u\" ,\"CFS_generation\":\"" UINT64_FORMAT "\","
-					"\"is_partial_copy\":\"%u\"",
-					file->is_cfs?1:0, file->generation, file->is_partial_copy?1:0);
-#endif
 		fprintf(out, "}\n");
 	}
 }
@@ -856,11 +855,6 @@ dir_read_file_list(const char *root, const char *file_txt)
 					is_datafile,
 					crc,
 					segno;
-#ifdef PGPRO_EE
-		uint64		generation,
-					is_partial_copy,
-					is_cfs;
-#endif
 		pgFile	   *file;
 
 		get_control_value(buf, "path", path, NULL, true);
@@ -874,11 +868,6 @@ dir_read_file_list(const char *root, const char *file_txt)
 		get_control_value(buf, "segno", NULL, &segno, false);
 		get_control_value(buf, "compress_alg", compress_alg_string, NULL, false);
 
-#ifdef PGPRO_EE
-		get_control_value(buf, "is_cfs", NULL, &is_cfs, false);
-		get_control_value(buf, "CFS_generation", NULL, &generation, false);
-		get_control_value(buf, "is_partial_copy", NULL, &is_partial_copy, false);
-#endif
 		if (root)
 			join_path_components(filepath, root, path);
 		else
@@ -894,11 +883,6 @@ dir_read_file_list(const char *root, const char *file_txt)
 		if (linked[0])
 			file->linked = pgut_strdup(linked);
 		file->segno = (int) segno;
-#ifdef PGPRO_EE
-		file->is_cfs = is_cfs ? true : false;
-		file->generation = generation;
-		file->is_partial_copy = is_partial_copy ? true : false;
-#endif
 
 		parray_append(files, file);
 	}

src/pg_probackup.c

@@ -17,7 +17,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-const char *PROGRAM_VERSION	= "2.0.4";
+const char *PROGRAM_VERSION	= "2.0.6";
 const char *PROGRAM_URL		= "https://github.com/postgrespro/pg_probackup";
 const char *PROGRAM_EMAIL	= "https://github.com/postgrespro/pg_probackup/issues";
 
@@ -48,6 +48,7 @@ char	   *replication_slot = NULL;
 bool		backup_logs = false;
 bool		smooth_checkpoint;
 bool		from_replica = false;
+bool		is_remote_backup = false;
 /* Wait timeout for WAL segment archiving */
 uint32		archive_timeout = 300;		/* default is 300 seconds */
 const char *master_db = NULL;
@@ -118,6 +119,8 @@ static pgut_option options[] =
 	{ 's', 15, "master-port",			&master_port,		SOURCE_CMDLINE, },
 	{ 's', 16, "master-user",			&master_user,		SOURCE_CMDLINE, },
 	{ 'u', 17, "replica-timeout",		&replica_timeout,	SOURCE_CMDLINE, },
+	/* TODO not completed feature. Make it unavailiable from user level
+	 { 'b', 18, "remote",				&is_remote_backup,	SOURCE_CMDLINE, }, */
 	/* restore options */
 	{ 's', 20, "time",					&target_time,		SOURCE_CMDLINE },
 	{ 's', 21, "xid",					&target_xid,		SOURCE_CMDLINE },

src/pg_probackup.h

@@ -27,6 +27,7 @@
 #include "storage/bufpage.h"
 #include "storage/checksum.h"
 #include "utils/pg_crc.h"
+#include "common/relpath.h"
 
 #include "utils/parray.h"
 #include "utils/pgut.h"
@@ -90,14 +91,13 @@ typedef struct pgFile
 	char	*linked;		/* path of the linked file */
 	bool	is_datafile;	/* true if the file is PostgreSQL data file */
 	char	*path;			/* path of the file */
-	char	*ptrack_path;	/* path of the ptrack fork of the relation */
+	Oid		tblspcOid;		/* tblspcOid extracted from path, if applicable */
+	Oid		dbOid;			/* dbOid extracted from path, if applicable */
+	Oid		relOid;			/* relOid extracted from path, if applicable */
+	char	*forkName;		/* forkName extracted from path, if applicable */
 	int		segno;			/* Segment number for ptrack */
 	bool	is_cfs;			/* Flag to distinguish files compressed by CFS*/
-	uint64	generation;		/* Generation of the compressed file.If generation
-							 * has changed, we cannot backup compressed file
-							 * partially. Has no sense if (is_cfs == false). */
-	bool	is_partial_copy; /* If the file was backed up via copy_file_partly().
-							  * Only applies to is_cfs files. */
+	bool	is_database;
 	CompressAlg compress_alg; /* compression algorithm applied to the file */
 	volatile uint32 lock;	/* lock for synchronization of parallel threads  */
 	datapagemap_t pagemap;	/* bitmap of pages updated since previous backup */
@@ -234,24 +234,6 @@ typedef union DataPage
 	char			data[BLCKSZ];
 } DataPage;
 
-/*
- * This struct and function definitions mirror ones from cfs.h, but doesn't use
- * atomic variables, since they are not allowed in frontend code.
- */
-typedef struct
-{
-	uint32 physSize;
-	uint32 virtSize;
-	uint32 usedSize;
-	uint32 lock;
-	pid_t	postmasterPid;
-	uint64	generation;
-	uint64	inodes[RELSEG_SIZE];
-} FileMap;
-
-extern FileMap* cfs_mmap(int md);
-extern int cfs_munmap(FileMap* map);
-
 /*
  * return pointer that exceeds the length of prefix from character string.
  * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz".
@@ -285,6 +267,7 @@ extern char *replication_slot;
 extern bool	smooth_checkpoint;
 extern uint32 archive_timeout;
 extern bool from_replica;
+extern bool is_remote_backup;
 extern const char *master_db;
 extern const char *master_host;
 extern const char *master_port;
@@ -432,16 +415,9 @@ extern bool backup_data_file(const char *from_root, const char *to_root,
 							 pgFile *file, XLogRecPtr prev_backup_start_lsn);
 extern void restore_data_file(const char *from_root, const char *to_root,
 							  pgFile *file, pgBackup *backup);
-extern void restore_compressed_file(const char *from_root,
-									const char *to_root, pgFile *file);
-extern bool backup_compressed_file_partially(pgFile *file,
-											 void *arg,
-											 size_t *skip_size);
 extern bool copy_file(const char *from_root, const char *to_root,
 					  pgFile *file);
 extern void copy_wal_file(const char *from_root, const char *to_root);
-extern bool copy_file_partly(const char *from_root, const char *to_root,
-				 pgFile *file, size_t skip_size);
 
 extern bool calc_file_checksum(pgFile *file);
 
@@ -469,7 +445,6 @@ extern void time2iso(char *buf, size_t len, time_t time);
 extern const char *status2str(BackupStatus status);
 extern void remove_trailing_space(char *buf, int comment_mark);
 extern void remove_not_digit(char *buf, size_t len, const char *str);
-extern XLogRecPtr get_last_ptrack_lsn(void);
 extern uint32 get_data_checksum_version(bool safe);
 extern char *base36enc(long unsigned int value);
 extern long unsigned int base36dec(const char *text);

src/restore.c

@@ -687,12 +687,7 @@ restore_files(void *arg)
 		 * copy the file from backup.
 		 */
 		if (file->is_datafile)
-		{
-			if (file->is_cfs)
-				restore_compressed_file(from_root, pgdata, file);
-			else
-				restore_data_file(from_root, pgdata, file, arguments->backup);
-		}
+			restore_data_file(from_root, pgdata, file, arguments->backup);
 		else
 			copy_file(from_root, pgdata, file);
 

src/util.c

@@ -81,24 +81,6 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
 	checkControlFile(ControlFile);
 }
 
-/*
- * Get lsn of the moment when ptrack was enabled the last time.
- */
-XLogRecPtr
-get_last_ptrack_lsn(void)
-{
-	char		*buffer;
-	size_t		size;
-	XLogRecPtr	lsn;
-
-	buffer = slurpFile(pgdata, "global/ptrack_control", &size, false);
-	if (buffer == NULL)
-		return 0;
-
-	lsn = *(XLogRecPtr *)buffer;
-	return lsn;
-}
-
 /*
  * Utility shared by backup and restore to fetch the current timeline
  * used by a node.

src/utils/pgut.c

@@ -1054,6 +1054,102 @@ pgut_connect_extended(const char *pghost, const char *pgport,
 	}
 }
 
+PGconn *
+pgut_connect_replication(const char *dbname)
+{
+	return pgut_connect_replication_extended(host, port, dbname, username, password);
+}
+
+PGconn *
+pgut_connect_replication_extended(const char *pghost, const char *pgport,
+						 const char *dbname, const char *pguser, const char *pwd)
+{
+	PGconn	   *tmpconn;
+	int			argcount = 7;	/* dbname, replication, fallback_app_name,
+								 * host, user, port, password */
+	int			i;
+	const char **keywords;
+	const char **values;
+
+	if (interrupted && !in_cleanup)
+		elog(ERROR, "interrupted");
+
+	i = 0;
+
+	keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
+	values = pg_malloc0((argcount + 1) * sizeof(*values));
+
+
+	keywords[i] = "dbname";
+	values[i] = "replication";
+	i++;
+	keywords[i] = "replication";
+	values[i] = "true";
+	i++;
+	keywords[i] = "fallback_application_name";
+	values[i] = PROGRAM_NAME;
+	i++;
+
+	if (pghost)
+	{
+		keywords[i] = "host";
+		values[i] = pghost;
+		i++;
+	}
+	if (pguser)
+	{
+		keywords[i] = "user";
+		values[i] = pguser;
+		i++;
+	}
+	if (pgport)
+	{
+		keywords[i] = "port";
+		values[i] = pgport;
+		i++;
+	}
+
+	/* Use (or reuse, on a subsequent connection) password if we have it */
+	if (password)
+	{
+		keywords[i] = "password";
+		values[i] = password;
+	}
+	else
+	{
+		keywords[i] = NULL;
+		values[i] = NULL;
+	}
+
+	for (;;)
+	{
+		tmpconn = PQconnectdbParams(keywords, values, true);
+
+
+		if (PQstatus(tmpconn) == CONNECTION_OK)
+		{
+			free(values);
+			free(keywords);
+			return tmpconn;
+		}
+
+		if (tmpconn && PQconnectionNeedsPassword(tmpconn) && prompt_password)
+		{
+			PQfinish(tmpconn);
+			prompt_for_password(username);
+			continue;
+		}
+
+		elog(ERROR, "could not connect to database %s: %s",
+			 dbname, PQerrorMessage(tmpconn));
+		PQfinish(tmpconn);
+		free(values);
+		free(keywords);
+		return NULL;
+	}
+}
+
+
 void
 pgut_disconnect(PGconn *conn)
 {

src/utils/pgut.h

@@ -118,6 +118,10 @@ extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata);
 extern PGconn *pgut_connect(const char *dbname);
 extern PGconn *pgut_connect_extended(const char *pghost, const char *pgport,
 									 const char *dbname, const char *login);
+extern PGconn *pgut_connect_replication(const char *dbname);
+extern PGconn *pgut_connect_replication_extended(const char *pghost, const char *pgport,
+									 const char *dbname, const char *login,
+									 const char *pwd);
 extern void pgut_disconnect(PGconn *conn);
 extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, const char **params);
 extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel);

tests/__init__.py

@@ -6,39 +6,37 @@ from . import init_test, option_test, show_test, \
     ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \
     ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \
     ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \
-    false_positive, replica, compression, page, ptrack, archive, \
-    cfs_backup, cfs_restore
+    false_positive, replica, compression, page, ptrack, archive
 
 
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()
-    suite.addTests(loader.loadTestsFromModule(init_test))
-    suite.addTests(loader.loadTestsFromModule(option_test))
-    suite.addTests(loader.loadTestsFromModule(show_test))
-    suite.addTests(loader.loadTestsFromModule(backup_test))
-    suite.addTests(loader.loadTestsFromModule(delete_test))
-    suite.addTests(loader.loadTestsFromModule(restore_test))
-    suite.addTests(loader.loadTestsFromModule(validate_test))
-    suite.addTests(loader.loadTestsFromModule(retention_test))
+#    suite.addTests(loader.loadTestsFromModule(init_test))
+#    suite.addTests(loader.loadTestsFromModule(option_test))
+#    suite.addTests(loader.loadTestsFromModule(show_test))
+#    suite.addTests(loader.loadTestsFromModule(backup_test))
+#    suite.addTests(loader.loadTestsFromModule(delete_test))
+#    suite.addTests(loader.loadTestsFromModule(restore_test))
+#    suite.addTests(loader.loadTestsFromModule(validate_test))
+#    suite.addTests(loader.loadTestsFromModule(retention_test))
     suite.addTests(loader.loadTestsFromModule(ptrack))
-    suite.addTests(loader.loadTestsFromModule(ptrack_clean))
-    suite.addTests(loader.loadTestsFromModule(ptrack_cluster))
-    suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace))
-    suite.addTests(loader.loadTestsFromModule(ptrack_recovery))
-    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum))
-    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen))
-    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility))
-    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full))
-    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate))
-    suite.addTests(loader.loadTestsFromModule(replica))
-    suite.addTests(loader.loadTestsFromModule(pgpro560))
-    suite.addTests(loader.loadTestsFromModule(pgpro589))
-    suite.addTests(loader.loadTestsFromModule(false_positive))
-    suite.addTests(loader.loadTestsFromModule(compression))
-    suite.addTests(loader.loadTestsFromModule(page))
-    suite.addTests(loader.loadTestsFromModule(archive))
-    suite.addTests(loader.loadTestsFromModule(cfs_backup))
-    suite.addTests(loader.loadTestsFromModule(cfs_restore))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_clean))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_cluster))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_recovery))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full))
+#    suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate))
+#    suite.addTests(loader.loadTestsFromModule(replica))
+#    suite.addTests(loader.loadTestsFromModule(pgpro560))
+#    suite.addTests(loader.loadTestsFromModule(pgpro589))
+#    suite.addTests(loader.loadTestsFromModule(false_positive))
+#    suite.addTests(loader.loadTestsFromModule(compression))
+#    suite.addTests(loader.loadTestsFromModule(page))
+#    suite.addTests(loader.loadTestsFromModule(archive))
+
     return suite
 
 # ToDo:

tests/expected/option_version.out

@@ -1 +1 @@
-pg_probackup 2.0.4
+pg_probackup 2.0.6

tests/helpers/ptrack_helpers.py

@@ -597,10 +597,10 @@ class ProbackupTest(object):
             pass
 
     def pgdata_content(self, directory):
-        """ return dict with directory content"""
+        """ return dict with directory content. TAKE IT AFTER CHECKPOINT or BACKUP"""
         dirs_to_ignore = ['pg_xlog', 'pg_wal', 'pg_log', 'pg_stat_tmp', 'pg_subtrans', 'pg_notify']
-        files_to_ignore = ['postmaster.pid', 'postmaster.opts']
-        suffixes_to_ignore = ('_ptrack', '_vm', '_fsm')
+        files_to_ignore = ['postmaster.pid', 'postmaster.opts', 'pg_internal.init']
+        suffixes_to_ignore = ('_ptrack', 'ptrack_control', 'pg_control', 'ptrack_init')
         directory_dict = {}
         directory_dict['pgdata'] = directory
         directory_dict['files'] = {}
@@ -615,14 +615,17 @@ class ProbackupTest(object):
         return directory_dict
 
     def compare_pgdata(self, original_pgdata, restored_pgdata):
-        """ return dict with directory content"""
+        """ return dict with directory content. DO IT BEFORE RECOVERY"""
         fail = False
         error_message = ''
         for file in original_pgdata['files']:
             if file in restored_pgdata['files']:
                 if original_pgdata['files'][file] != restored_pgdata['files'][file]:
-                    error_message += '\nChecksumm mismatch.\n File_old: {0}\n File_new: {1}'.format(
-                        os.path.join(original_pgdata['pgdata'], file), os.path.join(restored_pgdata['pgdata'], file))
+                    error_message += '\nChecksumm mismatch.\n File_old: {0}\n Checksumm_old: {1}\n File_new: {2}\n Checksumm_mew: {3}\n'.format(
+                        os.path.join(original_pgdata['pgdata'], file),
+                        original_pgdata['files'][file],
+                        os.path.join(restored_pgdata['pgdata'], file),
+                        restored_pgdata['files'][file])
                     fail = True
             else:
                 error_message += '\nFile dissappearance. File: {0}/{1}'.format(restored_pgdata['pgdata'], file)

tests/ptrack.py

@@ -12,7 +12,7 @@ module_name = 'ptrack'
 
 class PtrackBackupTest(ProbackupTest, unittest.TestCase):
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
     # @unittest.expectedFailure
     def test_ptrack_enable(self):
         """make ptrack without full backup, should result in error"""
@@ -42,7 +42,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
     def test_ptrack_stream(self):
         """make node, make full and ptrack stream backups, restore them and check data correctness"""
         self.maxDiff = None
@@ -51,7 +51,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
             )
 
         self.init_pb(backup_dir)
@@ -59,9 +59,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node.start()
 
         # FULL BACKUP
-        node.safe_psql(
-            "postgres",
-            "create sequence t_seq")
+        node.safe_psql("postgres", "create sequence t_seq")
         node.safe_psql(
             "postgres",
             "create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
@@ -74,6 +72,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
             "insert into t_heap select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(100,200) i")
         ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap")
         ptrack_backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream'])
+        pgdata = self.pgdata_content(node.data_dir)
 
         # Drop Node
         node.cleanup()
@@ -91,6 +90,8 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         self.assertIn("INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
             self.restore_node(backup_dir, 'node', node, backup_id=ptrack_backup_id, options=["-j", "4"]),
             '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
+        pgdata_restored = self.pgdata_content(node.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
         node.start()
         ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap")
         self.assertEqual(ptrack_result, ptrack_result_new)
@@ -98,7 +99,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
     def test_ptrack_archive(self):
         """make archive node, make full and ptrack backups, check data correctness in restored instance"""
         self.maxDiff = None
@@ -107,7 +108,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
             )
 
         self.init_pb(backup_dir)
@@ -128,6 +129,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
             "insert into t_heap select i as id, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(100,200) i")
         ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap")
         ptrack_backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack')
+        pgdata = self.pgdata_content(node.data_dir)
 
         # Drop Node
         node.cleanup()
@@ -145,6 +147,8 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         self.assertIn("INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
             self.restore_node(backup_dir, 'node', node, backup_id=ptrack_backup_id, options=["-j", "4"]),
             '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
+        pgdata_restored = self.pgdata_content(node.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
         node.start()
         ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap")
         self.assertEqual(ptrack_result, ptrack_result_new)
@@ -153,7 +157,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
     def test_ptrack_pgpro417(self):
         """Make  node, take full backup, take ptrack backup, delete ptrack backup. Try to take ptrack backup, which should fail"""
         self.maxDiff = None
@@ -162,7 +166,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
             )
 
         self.init_pb(backup_dir)
@@ -205,7 +209,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
     def test_page_pgpro417(self):
         """Make archive node, take full backup, take page backup, delete page backup. Try to take ptrack backup, which should fail"""
         self.maxDiff = None
@@ -214,7 +218,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
             )
 
         self.init_pb(backup_dir)
@@ -254,9 +258,9 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
                 '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
 
         # Clean after yourself
-        self.del_test_dir(module_name, fname)
+        # self.del_test_dir(module_name, fname)
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
     def test_full_pgpro417(self):
         """Make node, take two full backups, delete full second backup. Try to take ptrack backup, which should fail"""
         self.maxDiff = None
@@ -265,7 +269,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
             )
 
         self.init_pb(backup_dir)
@@ -306,7 +310,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
     def test_create_db(self):
         """Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense"""
         self.maxDiff = None
@@ -315,7 +319,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
             )
 
         self.init_pb(backup_dir)
@@ -323,36 +327,52 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node.start()
 
         # FULL BACKUP
-        node.safe_psql(
-            "postgres",
+        node.safe_psql("postgres",
             "create table t_heap as select i as id, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
         node.safe_psql("postgres", "SELECT * FROM t_heap")
         self.backup_node(backup_dir, 'node', node, options=["--stream"])
 
         # CREATE DATABASE DB1
-        node.safe_psql(
-            "postgres", "create database db1")
-#        node.safe_psql("db1", "create table t_heap as select i as id, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
-#        result =  node.safe_psql("db1", "select * from t_heap")
+        node.safe_psql("postgres", "create database db1")
+        node.safe_psql("db1", "create table t_heap as select i as id, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
 
         # PTRACK BACKUP
-        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
+        backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
         pgdata_content = self.pgdata_content(node.data_dir)
 
-        # DROP DATABASE DB1
-        #node.safe_psql(
-        #    "postgres", "drop database db1")
-        # SECOND PTRACK BACKUP
-        #self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
+        # RESTORE
         node_restored = self.make_simple_node(base_dir="{0}/{1}/node_restored".format(module_name, fname))
         node_restored.cleanup()
-
-        self.restore_node(backup_dir, 'node', node_restored, options=["-j", "4"])
+        # COMPARE PHYSICAL CONTENT
+        self.restore_node(backup_dir, 'node', node_restored, backup_id=backup_id, options=["-j", "4"])
         pgdata_content_new = self.pgdata_content(node_restored.data_dir)
         self.compare_pgdata(pgdata_content, pgdata_content_new)
-        result_new =  node_restored.safe_psql("db1", "select * from t_heap")
 
-        self.assertEqual(result, result_new)
+        # START RESTORED NODE
+        node_restored.append_conf("postgresql.auto.conf", "port = {0}".format(node_restored.port))
+        node_restored.start()
+        result_new =  node_restored.safe_psql("postgres", "select * from pg_class")
+
+
+        # DROP DATABASE DB1
+        node.safe_psql(
+            "postgres", "drop database db1")
+        # SECOND PTRACK BACKUP
+        backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
+        pgdata_content = self.pgdata_content(node.data_dir)
+
+        # RESTORE SECOND PTRACK BACKUP
+        node_restored.cleanup()
+        self.restore_node(backup_dir, 'node', node_restored, backup_id=backup_id, options=["-j", "4"])
+        # COMPARE PHYSICAL CONTENT
+        pgdata_content_new = self.pgdata_content(node_restored.data_dir)
+        self.compare_pgdata(pgdata_content, pgdata_content_new)
+
+
+        # START RESTORED NODE
+        node_restored.append_conf("postgresql.auto.conf", "port = {0}".format(node_restored.port))
+        node_restored.start()
+
         try:
             node_restored.safe_psql('db1', 'select 1')
             # we should die here because exception is what we expect to happen
@@ -365,7 +385,111 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
+    def test_alter_table_set_tablespace_ptrack(self):
+        """Make node, create tablespace with table, take full backup, alter tablespace location, take ptrack backup, restore database."""
+        self.maxDiff = None
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
+            )
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        node.start()
+
+        # FULL BACKUP
+        self.create_tblspace_in_node(node, 'somedata')
+        node.safe_psql(
+            "postgres",
+            "create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
+        node.safe_psql("postgres", "SELECT * FROM t_heap")
+        self.backup_node(backup_dir, 'node', node, options=["--stream"])
+
+        # ALTER TABLESPACE
+        self.create_tblspace_in_node(node, 'somedata_new')
+        node.safe_psql(
+            "postgres", "alter table t_heap set tablespace somedata_new")
+
+        # PTRACK BACKUP
+        result =  node.safe_psql("postgres", "select * from t_heap")
+        node.safe_psql("postgres", "select * from pg_class; checkpoint")
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
+        pgdata_content = self.pgdata_content(node.data_dir)
+        node.stop()
+
+        # RESTORE
+        node_restored = self.make_simple_node(base_dir="{0}/{1}/node_restored".format(module_name, fname))
+        node_restored.cleanup()
+
+        self.restore_node(backup_dir, 'node', node_restored, options=["-j", "4",
+            "-T", "{0}={1}".format(self.get_tblspace_path(node,'somedata'), self.get_tblspace_path(node_restored,'somedata')),
+            "-T", "{0}={1}".format(self.get_tblspace_path(node,'somedata_new'), self.get_tblspace_path(node_restored,'somedata_new'))
+            ])
+
+        # GET RESTORED PGDATA AND COMPARE
+        pgdata_content_new = self.pgdata_content(node_restored.data_dir)
+        self.compare_pgdata(pgdata_content, pgdata_content_new)
+        # START RESTORED NODE
+        node_restored.start()
+        result_new =  node.safe_psql("postgres", "select * from t_heap")
+
+        self.assertEqual(result, result_new, 'lost some data after restore')
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    #@unittest.skip("skip")
+    def test_alter_database_set_tablespace_ptrack(self):
+        """Make node, create tablespace with database, take full backup, alter tablespace location, take ptrack backup, restore database."""
+        self.maxDiff = None
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
+            )
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        node.start()
+
+        # FULL BACKUP
+        self.backup_node(backup_dir, 'node', node, options=["--stream"])
+
+        # CREATE TABLESPACE
+        self.create_tblspace_in_node(node, 'somedata')
+
+        # ALTER DATABASE
+        node.safe_psql("template1",
+            "alter database postgres set tablespace somedata")
+
+        # PTRACK BACKUP
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
+        pgdata_content = self.pgdata_content(node.data_dir)
+        node.stop()
+
+        # RESTORE
+        node_restored = self.make_simple_node(base_dir="{0}/{1}/node_restored".format(module_name, fname))
+        node_restored.cleanup()
+        self.restore_node(backup_dir, 'node', node_restored, options=["-j", "4",
+            "-T", "{0}={1}".format(self.get_tblspace_path(node,'somedata'), self.get_tblspace_path(node_restored,'somedata'))])
+
+        # GET PHYSICAL CONTENT
+        pgdata_content_new = self.pgdata_content(node_restored.data_dir)
+        # COMPARE PHYSICAL CONTENT
+        self.compare_pgdata(pgdata_content, pgdata_content_new)
+        # START RESTORED NODE
+        node_restored.start()
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    #@unittest.skip("skip")
     def test_drop_tablespace(self):
         """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup"""
         self.maxDiff = None
@@ -374,7 +498,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}
             )
 
         self.init_pb(backup_dir)
@@ -425,7 +549,7 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    # @unittest.skip("skip")
+    #@unittest.skip("skip")
     def test_alter_tablespace(self):
         """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup"""
         self.maxDiff = None
@@ -455,7 +579,9 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node.safe_psql(
             "postgres", "alter table t_heap set tablespace somedata")
         # FIRTS PTRACK BACKUP
+        node.safe_psql("postgres", "checkpoint")
         self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
+        pgdata_content = self.pgdata_content(node.data_dir)
 
         # Restore ptrack backup and check table consistency
         restored_node = self.make_simple_node(base_dir="{0}/{1}/restored_node".format(module_name, fname))
@@ -464,6 +590,10 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         tblspc_path_new = self.get_tblspace_path(restored_node, 'somedata_restored')
         self.restore_node(backup_dir, 'node', restored_node, options=[
             "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)])
+        result = node.safe_psql("postgres", "select * from t_heap")
+
+        pgdata_content_new = self.pgdata_content(restored_node.data_dir)
+        self.compare_pgdata(pgdata_content, pgdata_content_new)
         restored_node.append_conf("postgresql.auto.conf", "port = {0}".format(restored_node.port))
         restored_node.start()
 
@@ -476,11 +606,15 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
         node.safe_psql(
             "postgres", "alter table t_heap set tablespace pg_default")
         # SECOND PTRACK BACKUP
+        node.safe_psql("template1", "checkpoint")
         self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
+        pgdata_content = self.pgdata_content(node.data_dir)
 
         # Restore second ptrack backup and check table consistency
         self.restore_node(backup_dir, 'node', restored_node, options=[
             "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)])
+        pgdata_content_new = self.pgdata_content(restored_node.data_dir)
+        self.compare_pgdata(pgdata_content, pgdata_content_new)
         restored_node.append_conf("postgresql.auto.conf", "port = {0}".format(restored_node.port))
         restored_node.start()
         result_new = restored_node.safe_psql("postgres", "select * from t_heap")
@@ -488,3 +622,59 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_relation_with_multiple_segments(self):
+        """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup"""
+        self.maxDiff = None
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'ptrack_enable': 'on', 'fsync': 'off', 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB'}
+            )
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        node.start()
+
+#        self.create_tblspace_in_node(node, 'somedata')
+
+        # CREATE TABLE
+#        node.pgbench_init(scale=300, options=['--tablespace=somedata'])
+        pgbench = node.pgbench_init(scale=30)
+        # FULL BACKUP
+        self.backup_node(backup_dir, 'node', node, options=["--stream"])
+
+        pgbench = node.pgbench(options=['-T', '50', '-c', '2', '--no-vacuum'])
+        pgbench.wait()
+
+        # GET PHYSICAL CONTENT FROM NODE
+        # FIRTS PTRACK BACKUP
+        result = node.safe_psql("postgres", "select * from pgbench_accounts")
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
+        pgdata = self.pgdata_content(node.data_dir)
+
+        # RESTORE NODE
+#        self.restore_node(backup_dir, 'node', restored_node, options=[
+#            "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)])
+        restored_node = self.make_simple_node(base_dir="{0}/{1}/restored_node".format(module_name, fname))
+        restored_node.cleanup()
+#        tblspc_path = self.get_tblspace_path(node, 'somedata')
+#        tblspc_path_new = self.get_tblspace_path(restored_node, 'somedata_restored')
+
+        self.restore_node(backup_dir, 'node', restored_node, options=[
+            "-j", "4"])
+        # GET PHYSICAL CONTENT FROM NODE_RESTORED
+        pgdata_restored = self.pgdata_content(restored_node.data_dir)
+        self.compare_pgdata(pgdata, pgdata_restored)
+
+        restored_node.append_conf("postgresql.auto.conf", "port = {0}".format(restored_node.port))
+        restored_node.start()
+
+        result_new = restored_node.safe_psql("postgres", "select * from pgbench_accounts")
+        self.assertEqual(result, result_new)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)