mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2025-03-22 21:51:29 +02:00
Fix backup many segments of relation (big relations) with ptrack.
Add fail test for show problem in lost pages in backup process.
This commit is contained in:
stalkerg
parent
5698515521
commit
091fe0ada2
55
backup.c
55
backup.c
@ -110,9 +110,6 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
|
||||
*/
|
||||
current.tli = get_current_timeline(false);
|
||||
|
||||
if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK)
|
||||
pg_ptrack_clear();
|
||||
|
||||
/*
|
||||
* In differential backup mode, check if there is an already-validated
|
||||
* full backup on current timeline.
|
||||
@ -148,6 +145,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
|
||||
elog(ERROR, "backup_label does not exist in PGDATA.");
|
||||
}
|
||||
|
||||
if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK)
|
||||
pg_ptrack_clear();
|
||||
/*
|
||||
* List directories and symbolic links with the physical path to make
|
||||
* mkdirs.sh, then sort them in order of path. Omit $PGDATA.
|
||||
@ -915,18 +914,31 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
|
||||
path_len = strlen(file->path);
|
||||
if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0)
|
||||
{
|
||||
pgFile tmp_file;
|
||||
pgFile *search_file;
|
||||
//elog(WARNING, "Remove ptrack file from backup %s", file->path);
|
||||
tmp_file.path = pg_strdup(file->path);
|
||||
tmp_file.path[path_len-7] = '\0';
|
||||
search_file = *(pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
|
||||
if (search_file != NULL)
|
||||
{
|
||||
//elog(WARNING, "Finded main fork for ptrak:%s", search_file->path);
|
||||
search_file->ptrack_path = pg_strdup(file->path);
|
||||
pgFile **pre_search_file;
|
||||
int segno = 0;
|
||||
while(true) {
|
||||
pgFile tmp_file;
|
||||
tmp_file.path = pg_strdup(file->path);
|
||||
/* I hope segno not more than 999999 */
|
||||
if (segno > 0)
|
||||
sprintf(tmp_file.path+path_len-7, ".%d", segno);
|
||||
else
|
||||
tmp_file.path[path_len-7] = '\0';
|
||||
pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
|
||||
if (pre_search_file != NULL)
|
||||
{
|
||||
search_file = *pre_search_file;
|
||||
search_file->ptrack_path = pg_strdup(file->path);
|
||||
search_file->segno = segno;
|
||||
} else {
|
||||
pg_free(tmp_file.path);
|
||||
break;
|
||||
}
|
||||
pg_free(tmp_file.path);
|
||||
segno++;
|
||||
}
|
||||
free(tmp_file.path);
|
||||
|
||||
pgFileFree(file);
|
||||
parray_remove(list_file, i);
|
||||
i--;
|
||||
@ -1051,7 +1063,11 @@ void make_pagemap_from_ptrack(parray *files)
|
||||
if (p->ptrack_path != NULL)
|
||||
{
|
||||
DataPage page;
|
||||
char *flat_memory, *flat_mamory_cur;
|
||||
size_t flat_size = 0;
|
||||
size_t start_addr;
|
||||
struct stat st;
|
||||
|
||||
FILE *ptrack_file = fopen(p->ptrack_path, "r");
|
||||
if (ptrack_file == NULL)
|
||||
{
|
||||
@ -1060,16 +1076,21 @@ void make_pagemap_from_ptrack(parray *files)
|
||||
}
|
||||
|
||||
fstat(fileno(ptrack_file), &st);
|
||||
p->pagemap.bitmapsize = st.st_size-(st.st_size/BLCKSZ)*MAXALIGN(SizeOfPageHeaderData);
|
||||
p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize);
|
||||
flat_size = st.st_size-(st.st_size/BLCKSZ)*MAXALIGN(SizeOfPageHeaderData);
|
||||
flat_mamory_cur = flat_memory = pg_malloc(flat_size);
|
||||
|
||||
elog(LOG, "Start copy bitmap from ptrack:%s size:%i", p->ptrack_path, p->pagemap.bitmapsize);
|
||||
while(fread(page.data, BLCKSZ, 1, ptrack_file) == 1)
|
||||
{
|
||||
char *map = PageGetContents(page.data);
|
||||
memcpy(p->pagemap.bitmap, map, BLCKSZ - MAXALIGN(SizeOfPageHeaderData));
|
||||
memcpy(flat_memory, map, MAPSIZE);
|
||||
flat_mamory_cur += MAPSIZE;
|
||||
}
|
||||
fclose(ptrack_file);
|
||||
start_addr = (RELSEG_SIZE/8)*p->segno;
|
||||
p->pagemap.bitmapsize = start_addr+RELSEG_SIZE/8 > flat_size ? flat_size - start_addr : RELSEG_SIZE/8;
|
||||
p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize);
|
||||
memcpy(p->pagemap.bitmap, flat_memory+start_addr, p->pagemap.bitmapsize);
|
||||
pg_free(flat_memory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
2
data.c
2
data.c
@ -213,8 +213,6 @@ backup_data_file(const char *from_root, const char *to_root,
|
||||
return copy_file(from_root, to_root, file);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* if the page has not been modified since last backup, skip it */
|
||||
if (lsn && !XLogRecPtrIsInvalid(page_lsn) && page_lsn < *lsn)
|
||||
continue;
|
||||
|
||||
2
dir.c
2
dir.c
@ -88,6 +88,7 @@ pgFileNew(const char *path, bool omit_symlink)
|
||||
file->pagemap.bitmap = NULL;
|
||||
file->pagemap.bitmapsize = 0;
|
||||
file->ptrack_path = NULL;
|
||||
file->segno = 0;
|
||||
file->path = pgut_malloc(strlen(path) + 1);
|
||||
strcpy(file->path, path); /* enough buffer size guaranteed */
|
||||
|
||||
@ -549,6 +550,7 @@ dir_read_file_list(const char *root, const char *file_txt)
|
||||
file = (pgFile *) pgut_malloc(sizeof(pgFile));
|
||||
file->path = pgut_malloc((root ? strlen(root) + 1 : 0) + strlen(path) + 1);
|
||||
file->ptrack_path = NULL;
|
||||
file->segno = 0;
|
||||
file->pagemap.bitmap = NULL;
|
||||
file->pagemap.bitmapsize = 0;
|
||||
|
||||
|
||||
@ -42,6 +42,12 @@ OK: recovery-target-xid options works well.
|
||||
0
|
||||
0
|
||||
|
||||
###### RESTORE COMMAND TEST-0009 ######
|
||||
###### recovery to latest from full + ptrack backups with loads when full backup do ######
|
||||
0
|
||||
0
|
||||
0
|
||||
|
||||
###### RESTORE COMMAND TEST-0008 ######
|
||||
###### recovery with target inclusive false ######
|
||||
0
|
||||
|
||||
@ -59,6 +59,7 @@ typedef struct pgFile
|
||||
bool is_datafile; /* true if the file is PostgreSQL data file */
|
||||
char *path; /* path of the file */
|
||||
char *ptrack_path;
|
||||
int segno; /* Segment number for ptrack */
|
||||
datapagemap_t pagemap;
|
||||
} pgFile;
|
||||
|
||||
@ -81,6 +82,9 @@ typedef struct pgBackupRange
|
||||
(tm.tm_mon >= 0 && tm.tm_mon <= 11) && /* range check for tm_mon (0-23) */ \
|
||||
(tm.tm_year + 1900 >= 1900)) /* range check for tm_year(70-) */
|
||||
|
||||
/* Effective data size */
|
||||
#define MAPSIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData))
|
||||
|
||||
/* Backup status */
|
||||
/* XXX re-order ? */
|
||||
typedef enum BackupStatus
|
||||
|
||||
@ -170,6 +170,26 @@ psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches
|
||||
diff ${TEST_BASE}/TEST-0007-before.out ${TEST_BASE}/TEST-0007-after.out
|
||||
echo ''
|
||||
|
||||
echo '###### RESTORE COMMAND TEST-0009 ######'
|
||||
echo '###### recovery to latest from full + ptrack backups with loads when full backup do ######'
|
||||
init_backup
|
||||
pgbench_objs 0009
|
||||
pgbench -p ${TEST_PGPORT} -d pgbench -c 4 -T 8 > /dev/null 2>&1 &
|
||||
PGBENCH_PID=$!
|
||||
pg_arman backup -B ${BACKUP_PATH} -b full -j 4 -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0009-run.out 2>&1;echo $?
|
||||
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0009-run.out 2>&1
|
||||
#kill $PGBENCH_PID 2> /dev/null
|
||||
sleep 12
|
||||
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT count(*) FROM pgbench_history;" > ${TEST_BASE}/TEST-0009-count1.out
|
||||
pg_arman backup -B ${BACKUP_PATH} -b ptrack -j 4 -p ${TEST_PGPORT} -d postgres --verbose >> ${TEST_BASE}/TEST-0009-run.out 2>&1;echo $?
|
||||
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1
|
||||
pg_ctl stop -m immediate > /dev/null 2>&1
|
||||
pg_arman restore -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1;echo $?
|
||||
pg_ctl start -w -t 600 > /dev/null 2>&1
|
||||
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT count(*) FROM pgbench_history;" > ${TEST_BASE}/TEST-0009-count2.out
|
||||
diff ${TEST_BASE}/TEST-0009-count1.out ${TEST_BASE}/TEST-0009-count2.out
|
||||
echo ''
|
||||
|
||||
echo '###### RESTORE COMMAND TEST-0008 ######'
|
||||
echo '###### recovery with target inclusive false ######'
|
||||
init_backup
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user