Remove sanityChecks(). Fix add_files()

backup.c

@@ -126,9 +126,6 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
 	/* Initialize size summary */
 	current.data_bytes = 0;
 
-	/* do some checks on the node */
-	sanityChecks();
-
 	/*
 	 * Obtain current timeline by scanning control file, theh LSN
 	 * obtained at output of pg_start_backup or pg_stop_backup does
@@ -152,11 +149,9 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
 	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
 		current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
 	{
-		pgBackup   *prev_backup;
-
 		prev_backup = catalog_get_last_data_backup(backup_list, current.tli);
 		if (prev_backup == NULL)
-			elog(ERROR, "Timeline has changed since last full backup."
+			elog(ERROR, "Timeline has changed since last full backup. "
 						"Create new full backup before an incremental one.");
 	}
 
@@ -233,8 +228,7 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
 	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
 		current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
 	{
-		/* find last completed database backup */
+		Assert(prev_backup);
-		prev_backup = catalog_get_last_data_backup(backup_list, current.tli);
 		pgBackupGetPath(prev_backup, prev_file_txt, lengthof(prev_file_txt),
 			DATABASE_FILE_LIST);
 		prev_files = dir_read_file_list(pgdata, prev_file_txt);
@@ -1337,8 +1331,8 @@ backup_files(void *arg)
 static void
 add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
 {
-	parray	*list_file;
+	parray	   *list_file;
-	int		 i;
+	size_t		i;
 
 	list_file = parray_new();
 
@@ -1346,12 +1340,12 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
 	dir_list_file(list_file, root, true, true, add_root);
 
 	/* mark files that are possible datafile as 'datafile' */
-	for (i = 0; i < (int) parray_num(list_file); i++)
+	for (i = 0; i < parray_num(list_file); i++)
 	{
-		pgFile *file = (pgFile *) parray_get(list_file, i);
+		pgFile	   *file = (pgFile *) parray_get(list_file, i);
-		char *relative;
+		char	   *relative;
-		char *fname;
+		char	   *fname;
-		int path_len;
+		size_t		path_len;
 
 		/* data file must be a regular file */
 		if (!S_ISREG(file->mode))
@@ -1367,6 +1361,10 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
 
 		/* Get file name from path */
 		fname = last_dir_separator(relative);
+		if (fname == NULL)
+			fname = relative;
+		else
+			fname++;
 
 		/* Remove temp tables from the list */
 		if (fname[0] == 't' && isdigit(fname[1]))
@@ -1379,32 +1377,41 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
 
 		path_len = strlen(file->path);
 		/* Get link ptrack file to relations files */
-		if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0)
+		if (path_len > 6 &&
+			strncmp(file->path + (path_len - 6), "ptrack", 6) == 0)
 		{
-			pgFile *search_file;
+			pgFile	   *search_file;
-			pgFile **pre_search_file;
+			pgFile	  **pre_search_file;
-			int segno = 0;
+			int			segno = 0;
-			while(true) {
+
-				pgFile tmp_file;
+			while (true)
+			{
+				pgFile		tmp_file;
+
 				tmp_file.path = pg_strdup(file->path);
 
 				/* Segno fits into 6 digits since it is not more than 4000 */
 				if (segno > 0)
-					sprintf(tmp_file.path+path_len-7, ".%d", segno);
+					sprintf(tmp_file.path + path_len - 7, ".%d", segno);
 				else
-					tmp_file.path[path_len-7] = '\0';
+					tmp_file.path[path_len - 7] = '\0';
 
-				pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
+				pre_search_file = (pgFile **) parray_bsearch(list_file,
+															 &tmp_file,
+															 pgFileComparePath);
 
 				if (pre_search_file != NULL)
 				{
 					search_file = *pre_search_file;
 					search_file->ptrack_path = pg_strdup(file->path);
 					search_file->segno = segno;
-				} else {
+				}
+				else
+				{
 					pg_free(tmp_file.path);
 					break;
 				}
+
 				pg_free(tmp_file.path);
 				segno++;
 			}
@@ -1413,63 +1420,24 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
 			pgFileFree(file);
 			parray_remove(list_file, i);
 			i--;
-			continue;
 		}
-
 		/* compress map file it is not data file */
-		if (path_len > 4 && strncmp(file->path+(path_len-4), ".cfm", 4) == 0)
+		else if (path_len > 4 &&
-			continue;
+				 strncmp(file->path + (path_len - 4), ".cfm", 4) == 0)
-
-		/* name of data file start with digit */
-		if (fname == NULL)
-			fname = relative;
-		else
-			fname++;
-		if (!isdigit(fname[0]))
-			continue;
-
-		file->is_datafile = true;
 		{
-			int find_dot;
+			pgFile	   **pre_search_file;
-			int check_digit;
+			pgFile		tmp_file;
-			char *text_segno;
-			for(find_dot = path_len-1; file->path[find_dot] != '.' && find_dot >= 0; find_dot--);
-			if (find_dot <= 0)
-				continue;
 
-			text_segno = file->path + find_dot + 1;
-			for(check_digit=0; text_segno[check_digit] != '\0'; check_digit++)
-				if (!isdigit(text_segno[check_digit]))
-				{
-					check_digit = -1;
-					break;
-				}
-
-			if (check_digit == -1)
-				continue;
-
-			file->segno = (int) strtol(text_segno, NULL, 10);
-		}
-	}
-
-	/* mark cfs relations as not data */
-	for (i = 0; i < (int) parray_num(list_file); i++)
-	{
-		pgFile *file = (pgFile *) parray_get(list_file, i);
-		int path_len = (int) strlen(file->path);
-
-		if (path_len > 4 && strncmp(file->path+(path_len-4), ".cfm", 4) == 0)
-		{
-			pgFile **pre_search_file;
-			pgFile tmp_file;
 			tmp_file.path = pg_strdup(file->path);
-			tmp_file.path[path_len-4] = '\0';
+			tmp_file.path[path_len - 4] = '\0';
 			pre_search_file = (pgFile **) parray_bsearch(list_file,
-														 &tmp_file, pgFileComparePath);
+														 &tmp_file,
+														 pgFileComparePath);
 			if (pre_search_file != NULL)
 			{
-				FileMap* map;
+				FileMap	   *map;
-				int md = open(file->path, O_RDWR|PG_BINARY, 0);
+				int			md = open(file->path, O_RDWR|PG_BINARY, 0);
+
 				if (md < 0)
 					elog(ERROR, "add_files(). cannot open cfm file '%s'", file->path);
 
@@ -1485,16 +1453,51 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
 				(*pre_search_file)->is_datafile = false;
 
 				if (cfs_munmap(map) < 0)
-					elog(LOG, "add_files(). CFS failed to unmap file %s: %m", file->path);
+					elog(LOG, "add_files(). CFS failed to unmap file %s: %m",
+						 file->path);
 				if (close(md) < 0)
-					elog(LOG, "add_files(). CFS failed to close file %s: %m", file->path);
+					elog(LOG, "add_files(). CFS failed to close file %s: %m",
+						 file->path);
 			}
 			else
-				elog(ERROR, "corresponding segment '%s' is not found", tmp_file.path);
+				elog(ERROR, "corresponding segment '%s' is not found",
+					 tmp_file.path);
 
 			pg_free(tmp_file.path);
 		}
+		/* name of data file start with digit */
+		else if (isdigit(fname[0]))
+		{
+			int			find_dot;
+			int			check_digit;
+			char	   *text_segno;
+
+			file->is_datafile = true;
+
+			/*
+			 * Find segment number.
+			 */
+
+			for (find_dot = (int) path_len - 1;
+				 file->path[find_dot] != '.' && find_dot >= 0;
+				 find_dot--);
+			/* There is not segment number */
+			if (find_dot <= 0)
+				continue;
+
+			text_segno = file->path + find_dot + 1;
+			for (check_digit = 0; text_segno[check_digit] != '\0'; check_digit++)
+				if (!isdigit(text_segno[check_digit]))
+				{
+					check_digit = -1;
+					break;
+				}
+
+			if (check_digit != -1)
+				file->segno = (int) strtol(text_segno, NULL, 10);
+		}
 	}
+
 	parray_concat(files, list_file);
 }
 

delete.c

@@ -283,7 +283,7 @@ pgBackupDeleteFiles(pgBackup *backup)
 	parray_qsort(files, pgFileComparePathDesc);
 	for (i = 0; i < parray_num(files); i++)
 	{
-		pgFile *file = (pgFile *) parray_get(files, i);
+		pgFile	   *file = (pgFile *) parray_get(files, i);
 
 		/* print progress */
 		elog(LOG, "delete file(%zd/%lu) \"%s\"", i + 1,

dir.c

@@ -249,9 +249,6 @@ BlackListCompare(const void *str1, const void *str2)
  * List files, symbolic links and directories in the directory "root" and add
  * pgFile objects to "files".  We add "root" to "files" if add_root is true.
  *
- * If the sub-directory name is in "exclude" list, the sub-directory itself is
- * listed but the contents of the sub-directory is ignored.
- *
  * When omit_symlink is true, symbolic link is ignored and only file or
  * directory llnked to will be listed.
  */
@@ -259,40 +256,43 @@ void
 dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
 			  bool add_root)
 {
-	char path[MAXPGPATH];
+	parray	   *black_list = NULL;
-	char buf[MAXPGPATH * 2];
+	char		path[MAXPGPATH];
-	char black_item[MAXPGPATH * 2];
-	parray *black_list = NULL;
 
 	join_path_components(path, backup_path, PG_BLACK_LIST);
-	if (root && pgdata && strcmp(root, pgdata) == 0 &&
+	/* List files with black list */
-	    fileExists(path))
+	if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path))
 	{
-		FILE *black_list_file = NULL;
+		FILE	   *black_list_file = NULL;
+		char		buf[MAXPGPATH * 2];
+		char		black_item[MAXPGPATH * 2];
+
 		black_list = parray_new();
 		black_list_file = fopen(path, "r");
+
 		if (black_list_file == NULL)
-			elog(ERROR, "cannot open black_list: %s",
+			elog(ERROR, "cannot open black_list: %s", strerror(errno));
-				strerror(errno));
+
 		while (fgets(buf, lengthof(buf), black_list_file) != NULL)
 		{
 			join_path_components(black_item, pgdata, buf);
+
 			if (black_item[strlen(black_item) - 1] == '\n')
 				black_item[strlen(black_item) - 1] = '\0';
+
 			if (black_item[0] == '#' || black_item[0] == '\0')
 				continue;
+
 			parray_append(black_list, black_item);
 		}
+
 		fclose(black_list_file);
 		parray_qsort(black_list, BlackListCompare);
-		dir_list_file_internal(files, root, exclude, omit_symlink, add_root, black_list);
-		parray_qsort(files, pgFileComparePath);
-	}
-	else
-	{
-		dir_list_file_internal(files, root, exclude, omit_symlink, add_root, NULL);
-		parray_qsort(files, pgFileComparePath);
 	}
+
+	dir_list_file_internal(files, root, exclude, omit_symlink, add_root,
+						   black_list);
+	parray_qsort(files, pgFileComparePath);
 }
 
 void
@@ -561,7 +561,8 @@ dir_print_file_list(FILE *out, const parray *files, const char *root, const char
 			fprintf(out, " %s", timestamp);
 		}
 
-		fprintf(out, " %d %d\n", file->generation, file->is_partial_copy);
+		fprintf(out, " " UINT64_FORMAT " %d\n",
+				file->generation, file->is_partial_copy);
 	}
 }
 
@@ -681,30 +682,34 @@ dir_read_file_list(const char *root, const char *file_txt)
 	return files;
 }
 
-/* copy contents of directory from_root into to_root */
+/*
+ * Copy contents of directory from_root into to_root.
+ */
 void
 dir_copy_files(const char *from_root, const char *to_root)
 {
-	int		i;
+	size_t		i;
-	parray *files = parray_new();
+	parray	   *files = parray_new();
 
 	/* don't copy root directory */
 	dir_list_file(files, from_root, false, true, false);
 
 	for (i = 0; i < parray_num(files); i++)
 	{
-		pgFile *file = (pgFile *) parray_get(files, i);
+		pgFile	   *file = (pgFile *) parray_get(files, i);
 
 		if (S_ISDIR(file->mode))
 		{
-			char to_path[MAXPGPATH];
+			char		to_path[MAXPGPATH];
-			join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
+
+			join_path_components(to_path, to_root,
+								 file->path + strlen(from_root) + 1);
+
 			if (verbose && !check)
 				elog(LOG, "creating directory \"%s\"",
 					 file->path + strlen(from_root) + 1);
 			if (!check)
 				dir_create_dir(to_path, DIR_PERMISSION);
-			continue;
 		}
 		else if (S_ISREG(file->mode))
 		{

pg_probackup.h

@@ -70,10 +70,10 @@ typedef struct pgFile
 	char	*path;			/* path of the file */
 	char	*ptrack_path;
 	int		segno;			/* Segment number for ptrack */
-	int		generation;		/* Generation of compressed file.
+	uint64	generation;		/* Generation of compressed file.
 							 * -1 for non-compressed files */
-	int	is_partial_copy; /* for compressed files.
+	int		is_partial_copy; /* for compressed files.
-						  * 1 if backed up via copy_file_partly()  */
+							  * 1 if backed up via copy_file_partly()  */
 	volatile uint32 lock;
 	datapagemap_t pagemap;
 } pgFile;

tests/retention_test.py

@@ -63,18 +63,29 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
 		self.init_pb(node)
 		with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
 			conf.write("REDUNDANCY=1\n")
-			conf.write("WINDOW=2\n")
+			conf.write("WINDOW=1\n")
 
-		# All backups will be keeped
+		# Make backups to be purged
 		self.backup_pb(node)
 		self.backup_pb(node, backup_type="page")
+		# Make backup to be keeped
 		self.backup_pb(node)
+
+		backups = path.join(self.backup_dir(node), "backups")
+		days_delta = 5
+		for backup in listdir(backups):
+			with open(path.join(backups, backup, "backup.conf"), "a") as conf:
+				conf.write("RECOVERY_TIME='{:%Y-%m-%d %H:%M:%S}'\n".format(
+					datetime.now() - timedelta(days=days_delta)))
+				days_delta -= 1
+
+		# Make backup to be keeped
 		self.backup_pb(node, backup_type="page")		
 
 		self.assertEqual(len(self.show_pb(node)), 4)
 
 		# Purge backups
 		self.retention_purge_pb(node)
-		self.assertEqual(len(self.show_pb(node)), 4)
+		self.assertEqual(len(self.show_pb(node)), 2)
 
 		node.stop()

util.c

@@ -72,20 +72,6 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
 	checkControlFile(ControlFile);
 }
 
-void
-sanityChecks(void)
-{
-	ControlFileData	ControlFile;
-	char		   *buffer;
-	size_t			size;
-
-	/* First fetch file... */
-	buffer = slurpFile(pgdata, "global/pg_control", &size, false);
-	digestControlFile(&ControlFile, buffer, size);
-	pg_free(buffer);
-
-}
-
 XLogRecPtr
 get_last_ptrack_lsn(void)
 {
@@ -114,8 +100,9 @@ get_current_timeline(bool safe)
 
 	/* First fetch file... */
 	buffer = slurpFile(pgdata, "global/pg_control", &size, safe);
-	if (buffer == NULL)
+	if (safe && buffer == NULL)
 		return 0;
+
 	digestControlFile(&ControlFile, buffer, size);
 	pg_free(buffer);