Fix backup many segments of relation (big relations) with ptrack.

Add fail test for show problem in lost pages in backup process.

backup.c

@@ -110,9 +110,6 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
 	 */
 	current.tli = get_current_timeline(false);
 
-	if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK)
-		pg_ptrack_clear();
-
 	/*
 	 * In differential backup mode, check if there is an already-validated
 	 * full backup on current timeline.
@@ -148,6 +145,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
 		elog(ERROR, "backup_label does not exist in PGDATA.");
 	}
 
+	if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK)
+		pg_ptrack_clear();
 	/*
 	 * List directories and symbolic links with the physical path to make
 	 * mkdirs.sh, then sort them in order of path. Omit $PGDATA.
@@ -915,18 +914,31 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
 		path_len = strlen(file->path);
 		if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0)
 		{
-			pgFile tmp_file;
 			pgFile *search_file;
-			//elog(WARNING, "Remove ptrack file from backup %s", file->path);
+			pgFile **pre_search_file;
-			tmp_file.path = pg_strdup(file->path);
+			int segno = 0;
-			tmp_file.path[path_len-7] = '\0';
+			while(true) {
-			search_file = *(pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
+				pgFile tmp_file;
-			if (search_file != NULL)
+				tmp_file.path = pg_strdup(file->path);
-			{
+				/* I hope segno not more than 999999 */
-				//elog(WARNING, "Finded main fork for ptrak:%s", search_file->path);
+				if (segno > 0)
-				search_file->ptrack_path = pg_strdup(file->path);
+					sprintf(tmp_file.path+path_len-7, ".%d", segno);
+				else
+					tmp_file.path[path_len-7] = '\0';
+				pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
+				if (pre_search_file != NULL)
+				{
+					search_file = *pre_search_file;
+					search_file->ptrack_path = pg_strdup(file->path);
+					search_file->segno = segno;
+				} else {
+					pg_free(tmp_file.path);
+					break;
+				}
+				pg_free(tmp_file.path);
+				segno++;
 			}
-			free(tmp_file.path);
+
 			pgFileFree(file);
 			parray_remove(list_file, i);
 			i--;
@@ -1051,7 +1063,11 @@ void make_pagemap_from_ptrack(parray *files)
 		if (p->ptrack_path != NULL)
 		{
 			DataPage page;
+			char *flat_memory, *flat_mamory_cur;
+			size_t flat_size = 0;
+			size_t start_addr;
 			struct stat st;
+
 			FILE *ptrack_file = fopen(p->ptrack_path, "r");
 			if (ptrack_file == NULL)
 			{
@@ -1060,16 +1076,21 @@ void make_pagemap_from_ptrack(parray *files)
 			}
 
 			fstat(fileno(ptrack_file), &st);
-			p->pagemap.bitmapsize = st.st_size-(st.st_size/BLCKSZ)*MAXALIGN(SizeOfPageHeaderData);
+			flat_size = st.st_size-(st.st_size/BLCKSZ)*MAXALIGN(SizeOfPageHeaderData);
-			p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize);
+			flat_mamory_cur = flat_memory = pg_malloc(flat_size);
 
-			elog(LOG, "Start copy bitmap from ptrack:%s size:%i", p->ptrack_path, p->pagemap.bitmapsize);
 			while(fread(page.data, BLCKSZ, 1, ptrack_file) == 1)
 			{
 				char *map = PageGetContents(page.data);
-				memcpy(p->pagemap.bitmap, map, BLCKSZ - MAXALIGN(SizeOfPageHeaderData));
+				memcpy(flat_memory, map, MAPSIZE);
+				flat_mamory_cur += MAPSIZE;
 			}
 			fclose(ptrack_file);
+			start_addr = (RELSEG_SIZE/8)*p->segno;
+			p->pagemap.bitmapsize = start_addr+RELSEG_SIZE/8 > flat_size ? flat_size - start_addr : RELSEG_SIZE/8;
+			p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize);
+			memcpy(p->pagemap.bitmap, flat_memory+start_addr, p->pagemap.bitmapsize);
+			pg_free(flat_memory);
 		}
 	}
 }

data.c

@@ -213,8 +213,6 @@ backup_data_file(const char *from_root, const char *to_root,
 				return copy_file(from_root, to_root, file);
 			}
 
-			
-
 			/* if the page has not been modified since last backup, skip it */
 			if (lsn && !XLogRecPtrIsInvalid(page_lsn) && page_lsn < *lsn)
 				continue;

dir.c

@@ -88,6 +88,7 @@ pgFileNew(const char *path, bool omit_symlink)
 	file->pagemap.bitmap = NULL;
 	file->pagemap.bitmapsize = 0;
 	file->ptrack_path = NULL;
+	file->segno = 0;
 	file->path = pgut_malloc(strlen(path) + 1);
 	strcpy(file->path, path);		/* enough buffer size guaranteed */
 
@@ -549,6 +550,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 		file = (pgFile *) pgut_malloc(sizeof(pgFile));
 		file->path = pgut_malloc((root ? strlen(root) + 1 : 0) + strlen(path) + 1);
 		file->ptrack_path = NULL;
+		file->segno = 0;
 		file->pagemap.bitmap = NULL;
 		file->pagemap.bitmapsize = 0;
 

expected/restore.out

@@ -42,6 +42,12 @@ OK: recovery-target-xid options works well.
 0
 0
 
+###### RESTORE COMMAND TEST-0009 ######
+###### recovery to latest from full + ptrack backups with loads when full backup do ######
+0
+0
+0
+
 ###### RESTORE COMMAND TEST-0008 ######
 ###### recovery with target inclusive false ######
 0

pg_arman.h

@@ -59,6 +59,7 @@ typedef struct pgFile
 	bool	is_datafile;	/* true if the file is PostgreSQL data file */
 	char	*path;			/* path of the file */
 	char	*ptrack_path;
+	int		segno;			/* Segment number for ptrack */
 	datapagemap_t pagemap;
 } pgFile;
 
@@ -81,6 +82,9 @@ typedef struct pgBackupRange
 	 (tm.tm_mon >= 0 && tm.tm_mon <= 11) && 	/* range check for tm_mon (0-23)  */ \
 	 (tm.tm_year + 1900 >= 1900)) 			/* range check for tm_year(70-)    */
 
+/* Effective data size */
+#define MAPSIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData))
+
 /* Backup status */
 /* XXX re-order ? */
 typedef enum BackupStatus

sql/restore.sh

@@ -170,6 +170,26 @@ psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches
 diff ${TEST_BASE}/TEST-0007-before.out ${TEST_BASE}/TEST-0007-after.out
 echo ''
 
+echo '###### RESTORE COMMAND TEST-0009 ######'
+echo '###### recovery to latest from full + ptrack backups with loads when full backup do ######'
+init_backup
+pgbench_objs 0009
+pgbench -p ${TEST_PGPORT} -d pgbench -c 4 -T 8 > /dev/null 2>&1 &
+PGBENCH_PID=$!
+pg_arman backup -B ${BACKUP_PATH} -b full -j 4 -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0009-run.out 2>&1;echo $?
+pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0009-run.out 2>&1
+#kill $PGBENCH_PID 2> /dev/null
+sleep 12
+psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT count(*) FROM pgbench_history;" > ${TEST_BASE}/TEST-0009-count1.out
+pg_arman backup -B ${BACKUP_PATH} -b ptrack -j 4 -p ${TEST_PGPORT} -d postgres --verbose >> ${TEST_BASE}/TEST-0009-run.out 2>&1;echo $?
+pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1
+pg_ctl stop -m immediate > /dev/null 2>&1
+pg_arman restore -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1;echo $?
+pg_ctl start -w -t 600 > /dev/null 2>&1
+psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT count(*) FROM pgbench_history;" > ${TEST_BASE}/TEST-0009-count2.out
+diff ${TEST_BASE}/TEST-0009-count1.out ${TEST_BASE}/TEST-0009-count2.out
+echo ''
+
 echo '###### RESTORE COMMAND TEST-0008 ######'
 echo '###### recovery with target inclusive false ######'
 init_backup