1
0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-02-08 14:28:36 +02:00

First version of ptrack support.

This commit is contained in:
Zhuravlev Uriy aka stalkerg 2016-02-27 21:07:55 +03:00
parent 7cbbdf0fc4
commit 9c475eccbf
13 changed files with 201 additions and 30 deletions

103
backup.c
View File

@ -19,6 +19,8 @@
#include "libpq/pqsignal.h"
#include "pgut/pgut-port.h"
#include "storage/bufpage.h"
#include "datapagemap.h"
/* wait 10 sec until WAL archive complete */
#define TIMEOUT_ARCHIVE 10
@ -44,6 +46,7 @@ static void pg_stop_backup(pgBackup *backup);
static bool pg_is_standby(void);
static void get_lsn(PGresult *res, XLogRecPtr *lsn);
static void get_xid(PGresult *res, uint32 *xid);
static void pg_ptrack_clear(void);
static void add_files(parray *files, const char *root, bool add_root, bool is_pgdata);
static void create_file_list(parray *files,
@ -52,6 +55,7 @@ static void create_file_list(parray *files,
const char *prefix,
bool is_append);
static void wait_for_archive(pgBackup *backup, const char *sql);
static void make_pagemap_from_ptrack(parray *files);
/*
* Take a backup of database and return the list of files backed up.
@ -96,7 +100,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
* In differential backup mode, check if there is an already-validated
* full backup on current timeline.
*/
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
{
pgBackup *prev_backup;
@ -156,7 +161,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
* To take differential backup, the file list of the last completed database
* backup is needed.
*/
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
{
/* find last completed database backup */
prev_backup = catalog_get_last_data_backup(backup_list, current.tli);
@ -209,15 +215,24 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
current.start_lsn);
}
if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
{
parray_qsort(backup_files_list, pgFileComparePathDesc);
make_pagemap_from_ptrack(backup_files_list);
}
backup_files(pgdata, path, backup_files_list, prev_files, lsn, NULL);
/* notify end of backup */
/* Clear ptrack files after backup */
if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
pg_ptrack_clear();
/* Notify end of backup */
pg_stop_backup(&current);
/* create file list */
/* Create file list */
create_file_list(backup_files_list, pgdata, DATABASE_FILE_LIST, NULL, false);
/* print summary of size of backup mode files */
/* Print summary of size of backup mode files */
for (i = 0; i < parray_num(backup_files_list); i++)
{
pgFile *file = (pgFile *) parray_get(backup_files_list, i);
@ -228,7 +243,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
* amount of data written counts while for an differential
* backup only the data read counts.
*/
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
current.data_bytes += file->read_size;
else if (current.backup_mode == BACKUP_MODE_FULL)
current.data_bytes += file->size;
@ -332,7 +348,8 @@ do_backup(pgBackupOption bkupopt)
/* Database data */
if (current.backup_mode == BACKUP_MODE_FULL ||
current.backup_mode == BACKUP_MODE_DIFF_PAGE)
current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
total_read += current.data_bytes;
if (total_read == 0)
@ -435,6 +452,17 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup)
disconnect();
}
static void
pg_ptrack_clear(void)
{
PGresult *res;
reconnect();
res = execute("select pg_ptrack_clear()", 0, NULL);
PQclear(res);
disconnect();
}
static void
wait_for_archive(pgBackup *backup, const char *sql)
{
@ -806,6 +834,7 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
pgFile *file = (pgFile *) parray_get(list_file, i);
char *relative;
char *fname;
int path_len;
/* data file must be a regular file */
if (!S_ISREG(file->mode))
@ -819,6 +848,27 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
!path_is_prefix_of_path("pg_tblspc", relative))
continue;
path_len = strlen(file->path);
if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0)
{
pgFile tmp_file;
pgFile *search_file;
//elog(WARNING, "Remove ptrack file from backup %s", file->path);
tmp_file.path = pg_strdup(file->path);
tmp_file.path[path_len-7] = '\0';
search_file = *(pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
if (search_file != NULL)
{
//elog(WARNING, "Finded main fork for ptrak:%s", search_file->path);
search_file->ptrack_path = pg_strdup(file->path);
}
free(tmp_file.path);
pgFileFree(file);
parray_remove(list_file, i);
i--;
continue;
}
/* name of data file start with digit */
fname = last_dir_separator(relative);
if (fname == NULL)
@ -927,3 +977,42 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
pg_free(path);
pg_free(rel_path);
}
void make_pagemap_from_ptrack(parray *files)
{
int i;
for (i = 0; i < parray_num(files); i++)
{
pgFile *p = (pgFile *) parray_get(files, i);
if (p->ptrack_path != NULL)
{
DataPage page;
int i;
struct stat st;
FILE *ptrack_file = fopen(p->ptrack_path, "r");
if (ptrack_file == NULL)
{
elog(ERROR, "cannot open ptrack file \"%s\": %s", p->ptrack_path,
strerror(errno));
}
elog(LOG, "Start copy bitmap from ptrack:%s", p->ptrack_path);
fstat(fileno(ptrack_file), &st);
p->pagemap.bitmapsize = st.st_size-(st.st_size/BLCKSZ)*MAXALIGN(SizeOfPageHeaderData);
p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize);
while(fread(page.data, BLCKSZ, 1, ptrack_file) == BLCKSZ)
{
char *map = PageGetContents(page.data);
memcpy(p->pagemap.bitmap, map, BLCKSZ-MAXALIGN(SizeOfPageHeaderData));
}
fclose(ptrack_file);
for(i = 0; i < p->pagemap.bitmapsize; i++)
if (p->pagemap.bitmap[i] != 0)
goto end_loop;
pg_free(p->pagemap.bitmap);
p->pagemap.bitmapsize = 0;
}
end_loop:;
}
}

View File

@ -252,6 +252,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli)
if (backup->status == BACKUP_STATUS_OK &&
backup->tli == tli &&
(backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
backup->backup_mode == BACKUP_MODE_DIFF_PTRACK ||
backup->backup_mode == BACKUP_MODE_FULL))
return backup;
}
@ -286,7 +287,7 @@ pgBackupCreateDir(pgBackup *backup)
void
pgBackupWriteConfigSection(FILE *out, pgBackup *backup)
{
static const char *modes[] = { "", "PAGE", "FULL"};
static const char *modes[] = { "", "PAGE", "PTRACK", "FULL"};
fprintf(out, "# configuration\n");
fprintf(out, "BACKUP_MODE=%s\n", modes[backup->backup_mode]);
@ -478,6 +479,8 @@ parse_backup_mode(const char *value)
return BACKUP_MODE_FULL;
else if (len > 0 && pg_strncasecmp("page", v, strlen("page")) == 0)
return BACKUP_MODE_DIFF_PAGE;
else if (len > 0 && pg_strncasecmp("ptrack", v, strlen("ptrack")) == 0)
return BACKUP_MODE_DIFF_PTRACK;
/* Backup mode is invalid, so leave with an error */
elog(ERROR, "invalid backup-mode \"%s\"", value);

7
data.c
View File

@ -18,12 +18,6 @@
#include "storage/block.h"
#include "storage/bufpage.h"
typedef union DataPage
{
PageHeaderData page_data;
char data[BLCKSZ];
} DataPage;
typedef struct BackupPageHeader
{
BlockNumber block; /* block number */
@ -184,7 +178,6 @@ backup_data_file(const char *from_root, const char *to_root,
else
{
datapagemap_iterator_t *iter;
iter = datapagemap_iterate(&file->pagemap);
while (datapagemap_next(iter, &blknum))
{

4
dir.c
View File

@ -91,6 +91,7 @@ pgFileNew(const char *path, bool omit_symlink)
file->linked = NULL;
file->pagemap.bitmap = NULL;
file->pagemap.bitmapsize = 0;
file->ptrack_path = NULL;
file->path = pgut_malloc(strlen(path) + 1);
strcpy(file->path, path); /* enough buffer size guaranteed */
@ -172,6 +173,8 @@ pgFileFree(void *file)
return;
free(((pgFile *)file)->linked);
free(((pgFile *)file)->path);
if (((pgFile *)file)->ptrack_path != NULL)
free(((pgFile *)file)->ptrack_path);
free(file);
}
@ -549,6 +552,7 @@ dir_read_file_list(const char *root, const char *file_txt)
file = (pgFile *) pgut_malloc(sizeof(pgFile));
file->path = pgut_malloc((root ? strlen(root) + 1 : 0) + strlen(path) + 1);
file->ptrack_path = NULL;
file->pagemap.bitmap = NULL;
file->pagemap.bitmapsize = 0;

View File

@ -32,3 +32,9 @@ page-level backup without validated full backup
1
0
0
###### BACKUP COMMAND TEST-0006 ######
###### ptrack backup mode ######
0
0
2
6

View File

@ -30,6 +30,19 @@ OK: recovery.conf has the given target timeline.
OK: recovery-target-xid options works well.
###### RESTORE COMMAND TEST-0006 ######
###### recovery to latest from full + ptrack backups ######
0
0
0
###### RESTORE COMMAND TEST-0007 ######
###### recovery to latest from full + ptrack + ptrack backups ######
0
0
0
0
###### RESTORE COMMAND TEST-0008 ######
###### recovery with target inclusive false ######
0
0

View File

@ -21,6 +21,8 @@
#include "utils/pg_crc.h"
#include "parray.h"
#include "datapagemap.h"
#include "storage/bufpage.h"
#include "storage/block.h"
/* Query to fetch current transaction ID */
#define TXID_CURRENT_SQL "SELECT txid_current();"
@ -55,7 +57,8 @@ typedef struct pgFile
pg_crc32 crc; /* CRC value of the file, regular file only */
char *linked; /* path of the linked file */
bool is_datafile; /* true if the file is PostgreSQL data file */
char *path; /* path of the file */
char *path; /* path of the file */
char *ptrack_path;
datapagemap_t pagemap;
} pgFile;
@ -96,6 +99,7 @@ typedef enum BackupMode
{
BACKUP_MODE_INVALID,
BACKUP_MODE_DIFF_PAGE, /* differential page backup */
BACKUP_MODE_DIFF_PTRACK, /* differential page backup with ptrack system*/
BACKUP_MODE_FULL /* full backup */
} BackupMode;
@ -161,6 +165,11 @@ typedef struct pgRecoveryTarget
bool recovery_target_inclusive;
} pgRecoveryTarget;
typedef union DataPage
{
PageHeaderData page_data;
char data[BLCKSZ];
} DataPage;
/*
* return pointer that exceeds the length of prefix from character string.

View File

@ -166,7 +166,8 @@ base_backup_found:
continue;
/* use database backup only */
if (backup->backup_mode != BACKUP_MODE_DIFF_PAGE)
if (backup->backup_mode != BACKUP_MODE_DIFF_PAGE &&
backup->backup_mode != BACKUP_MODE_DIFF_PTRACK)
continue;
/* is the backup is necessary for restore to target timeline ? */

4
show.c
View File

@ -176,7 +176,7 @@ show_backup_list(FILE *out, parray *backup_list, bool show_all)
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup;
const char *modes[] = { "", "PAGE", "FULL"};
const char *modes[] = { "", "PAGE", "PTRACK", "FULL"};
TimeLineID parent_tli;
char timestamp[20];
char duration[20] = "----";
@ -204,7 +204,7 @@ show_backup_list(FILE *out, parray *backup_list, bool show_all)
/* Get parent timeline before printing */
parent_tli = get_parent_tli(backup->tli);
fprintf(out, "%-19s %-4s %10d %10d %5s %6s %s \n",
fprintf(out, "%-19s %-6s %10d %10d %5s %6s %s \n",
timestamp,
modes[backup->backup_mode],
backup->tli,

View File

@ -74,6 +74,17 @@ pg_arman show -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0005.log 2>&1
grep OK ${TEST_BASE}/TEST-0005.log | grep FULL | wc -l | sed 's/^ *//'
grep ERROR ${TEST_BASE}/TEST-0005.log | grep INCR | wc -l | sed 's/^ *//'
echo '###### BACKUP COMMAND TEST-0006 ######'
echo '###### ptrack backup mode ######'
init_catalog
pg_arman backup -B ${BACKUP_PATH} -b full -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0006-run.log 2>&1;echo $?
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.log 2>&1
pg_arman backup -B ${BACKUP_PATH} -b ptrack -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0006-run.log 2>&1;echo $?
pg_arman validate -B ${BACKUP_PATH} >> ${TEST_BASE}/TEST-0006-run.log 2>&1
pg_arman show -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0006.log 2>&1
grep -c OK ${TEST_BASE}/TEST-0006.log
grep OK ${TEST_BASE}/TEST-0006.log | sed -e 's@[^-]@@g' | wc -c | sed 's/^ *//'
# cleanup
## clean up the temporal test data
pg_ctl stop -m immediate -D ${PGDATA_PATH} > /dev/null 2>&1

View File

@ -73,6 +73,7 @@ wal_level = hot_standby
wal_log_hints = on
archive_mode = on
archive_command = 'cp %p ${ARCLOG_PATH}/%f'
ptrack_enable = on
EOF
# start PostgreSQL

View File

@ -133,15 +133,53 @@ fi
echo ''
echo '###### RESTORE COMMAND TEST-0006 ######'
echo '###### recovery with target inclusive false ######'
echo '###### recovery to latest from full + ptrack backups ######'
init_backup
pgbench_objs 0006
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "CREATE TABLE tbl0006 (a text);" > /dev/null 2>&1
pg_arman backup -B ${BACKUP_PATH} -b full -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0006-run.out 2>&1;echo $?
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1
pgbench -p ${TEST_PGPORT} -d pgbench > /dev/null 2>&1
pg_arman backup -B ${BACKUP_PATH} -b ptrack -p ${TEST_PGPORT} -d postgres --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1;echo $?
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches;" > ${TEST_BASE}/TEST-0006-before.out
pg_ctl stop -m immediate > /dev/null 2>&1
pg_arman restore -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1;echo $?
pg_ctl start -w -t 600 > /dev/null 2>&1
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches;" > ${TEST_BASE}/TEST-0006-after.out
diff ${TEST_BASE}/TEST-0006-before.out ${TEST_BASE}/TEST-0006-after.out
echo ''
echo '###### RESTORE COMMAND TEST-0007 ######'
echo '###### recovery to latest from full + ptrack + ptrack backups ######'
init_backup
pgbench_objs 0007
pg_arman backup -B ${BACKUP_PATH} -b full -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0007-run.out 2>&1;echo $?
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0007-run.out 2>&1
pgbench -p ${TEST_PGPORT} -d pgbench > /dev/null 2>&1
pg_arman backup -B ${BACKUP_PATH} -b ptrack -p ${TEST_PGPORT} -d postgres --verbose >> ${TEST_BASE}/TEST-0007-run.out 2>&1;echo $?
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0007-run.out 2>&1
pgbench -p ${TEST_PGPORT} -d pgbench > /dev/null 2>&1
pg_arman backup -B ${BACKUP_PATH} -b ptrack -p ${TEST_PGPORT} -d postgres --verbose >> ${TEST_BASE}/TEST-0007-run.out 2>&1;echo $?
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0007-run.out 2>&1
pg_arman show -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0007-show.out 2>&1
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches;" > ${TEST_BASE}/TEST-0007-before.out
pg_ctl stop -m immediate > /dev/null 2>&1
pg_arman restore -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0007-run.out 2>&1;echo $?
pg_ctl start -w -t 600 > /dev/null 2>&1
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches;" > ${TEST_BASE}/TEST-0007-after.out
diff ${TEST_BASE}/TEST-0007-before.out ${TEST_BASE}/TEST-0007-after.out
echo ''
echo '###### RESTORE COMMAND TEST-0008 ######'
echo '###### recovery with target inclusive false ######'
init_backup
pgbench_objs 0008
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "CREATE TABLE tbl0008 (a text);" > /dev/null 2>&1
pg_arman backup -B ${BACKUP_PATH} -b full -p ${TEST_PGPORT} -d postgres --verbose > ${TEST_BASE}/TEST-0008-run.out 2>&1;echo $?
pg_arman validate -B ${BACKUP_PATH} --verbose >> ${TEST_BASE}/TEST-0003-run.out 2>&1
pgbench -p ${TEST_PGPORT} pgbench > /dev/null 2>&1
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches;" > ${TEST_BASE}/TEST-0006-before.out
TARGET_XID=`psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -tAq -c "INSERT INTO tbl0006 VALUES ('inserted') RETURNING (xmin);"`
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches;" > ${TEST_BASE}/TEST-0008-before.out
TARGET_XID=`psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -tAq -c "INSERT INTO tbl0008 VALUES ('inserted') RETURNING (xmin);"`
pgbench -p ${TEST_PGPORT} -d pgbench > /dev/null 2>&1
# Enforce segment to be archived to ensure that recovery goes up to the
# wanted point. There is no way to ensure that all segments needed have
@ -149,12 +187,12 @@ pgbench -p ${TEST_PGPORT} -d pgbench > /dev/null 2>&1
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c 'SELECT pg_switch_xlog()' > /dev/null 2>&1
# Fast mode is used to ensure that the last segment is archived as well.
pg_ctl stop -m fast > /dev/null 2>&1
pg_arman restore -B ${BACKUP_PATH} --recovery-target-xid="${TARGET_XID}" --recovery-target-inclusive=false --verbose >> ${TEST_BASE}/TEST-0006-run.out 2>&1;echo $?
pg_arman restore -B ${BACKUP_PATH} --recovery-target-xid="${TARGET_XID}" --recovery-target-inclusive=false --verbose >> ${TEST_BASE}/TEST-0008-run.out 2>&1;echo $?
pg_ctl start -w -t 600 > /dev/null 2>&1
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches;" > ${TEST_BASE}/TEST-0006-after.out
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM tbl0006;" > ${TEST_BASE}/TEST-0006-tbl.dump
diff ${TEST_BASE}/TEST-0006-before.out ${TEST_BASE}/TEST-0006-after.out
if grep "inserted" ${TEST_BASE}/TEST-0006-tbl.dump > /dev/null ; then
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM pgbench_branches;" > ${TEST_BASE}/TEST-0008-after.out
psql --no-psqlrc -p ${TEST_PGPORT} -d pgbench -c "SELECT * FROM tbl0008;" > ${TEST_BASE}/TEST-0008-tbl.dump
diff ${TEST_BASE}/TEST-0008-before.out ${TEST_BASE}/TEST-0008-after.out
if grep "inserted" ${TEST_BASE}/TEST-0008-tbl.dump > /dev/null ; then
echo 'NG: recovery-target-inclusive=false does not work well.'
else
echo 'OK: recovery-target-inclusive=false works well.'
@ -163,6 +201,7 @@ else
fi
echo ''
# clean up the temporal test data
pg_ctl stop -m immediate > /dev/null 2>&1
rm -fr ${PGDATA_PATH}

View File

@ -83,7 +83,8 @@ pgBackupValidate(pgBackup *backup,
if (!for_get_timeline)
{
if (backup->backup_mode == BACKUP_MODE_FULL ||
backup->backup_mode == BACKUP_MODE_DIFF_PAGE)
backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)
elog(INFO, "validate: %s backup and archive log files by %s",
timestamp, (size_only ? "SIZE" : "CRC"));
}
@ -91,7 +92,8 @@ pgBackupValidate(pgBackup *backup,
if (!check)
{
if (backup->backup_mode == BACKUP_MODE_FULL ||
backup->backup_mode == BACKUP_MODE_DIFF_PAGE)
backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)
{
elog(LOG, "database files...");
pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR);