mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2025-02-03 14:01:57 +02:00
Replace incremental backup by page-level differential backup
This will allow the introduction of file-level differential backup.
This commit is contained in:
Michael Paquier
parent
a1de8d6145
commit
db249c800e
4
README
4
README
@ -2,11 +2,11 @@ pg_arman
|
||||
========
|
||||
|
||||
pg_arman is a backup and recovery manager for PostgreSQL servers able to do
|
||||
incremental and full backup as well as restore a cluster to a
|
||||
differential and full backup as well as restore a cluster to a
|
||||
state defined by a given recovery target. It is designed to perform
|
||||
periodic backups of an existing PostgreSQL server, combined with WAL
|
||||
archives to provide a way to recover a server in case of failure of
|
||||
server because of a reason or another. Its incremental backup
|
||||
server because of a reason or another. Its differential backup
|
||||
facility reduces the amount of data necessary to be taken between
|
||||
two consecutive backups.
|
||||
|
||||
|
||||
22
backup.c
22
backup.c
@ -93,17 +93,17 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
|
||||
current.tli = get_current_timeline();
|
||||
|
||||
/*
|
||||
* In incremental backup mode, check if there is an already-validated
|
||||
* In differential backup mode, check if there is an already-validated
|
||||
* full backup on current timeline.
|
||||
*/
|
||||
if (current.backup_mode == BACKUP_MODE_INCREMENTAL)
|
||||
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
|
||||
{
|
||||
pgBackup *prev_backup;
|
||||
|
||||
prev_backup = catalog_get_last_data_backup(backup_list, current.tli);
|
||||
if (prev_backup == NULL)
|
||||
elog(ERROR_SYSTEM, _("Valid full backup not found for "
|
||||
"incremental backup. Either create a full backup "
|
||||
"differential backup. Either create a full backup "
|
||||
"or validate existing one."));
|
||||
}
|
||||
|
||||
@ -154,10 +154,10 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
|
||||
files = NULL;
|
||||
|
||||
/*
|
||||
* To take incremental backup, the file list of the last completed database
|
||||
* To take differential backup, the file list of the last completed database
|
||||
* backup is needed.
|
||||
*/
|
||||
if (current.backup_mode == BACKUP_MODE_INCREMENTAL)
|
||||
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
|
||||
{
|
||||
pgBackup *prev_backup;
|
||||
|
||||
@ -287,7 +287,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
|
||||
elog(ERROR_SYSTEM, _("tablespace storage directory doesn't exist: %s"), mp);
|
||||
|
||||
/*
|
||||
* create the previous backup file list to take incremental backup
|
||||
* create the previous backup file list to take differential backup
|
||||
* from the snapshot volume.
|
||||
*/
|
||||
if (prev_files != NULL)
|
||||
@ -392,10 +392,10 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
|
||||
continue;
|
||||
/*
|
||||
* Count only the amount of data. For a full backup, the total
|
||||
* amount of data written counts while for an incremental
|
||||
* amount of data written counts while for an differential
|
||||
* backup only the data read counts.
|
||||
*/
|
||||
if (current.backup_mode == BACKUP_MODE_INCREMENTAL)
|
||||
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
|
||||
current.data_bytes += file->read_size;
|
||||
else if (current.backup_mode == BACKUP_MODE_FULL)
|
||||
current.data_bytes += file->size;
|
||||
@ -514,7 +514,7 @@ do_backup(pgBackupOption bkupopt)
|
||||
|
||||
/* Database data */
|
||||
if (current.backup_mode == BACKUP_MODE_FULL ||
|
||||
current.backup_mode == BACKUP_MODE_INCREMENTAL)
|
||||
current.backup_mode == BACKUP_MODE_DIFF_PAGE)
|
||||
total_read += current.data_bytes;
|
||||
|
||||
if (total_read == 0)
|
||||
@ -821,7 +821,9 @@ backup_cleanup(bool fatal, void *userdata)
|
||||
}
|
||||
}
|
||||
|
||||
/* take incremental backup. */
|
||||
/*
|
||||
* Take differential backup at page level.
|
||||
*/
|
||||
static void
|
||||
backup_files(const char *from_root,
|
||||
const char *to_root,
|
||||
|
||||
12
catalog.c
12
catalog.c
@ -256,11 +256,11 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli)
|
||||
|
||||
/*
|
||||
* We need completed database backup in the case of a full or
|
||||
* incremental backup on current timeline.
|
||||
* differential backup on current timeline.
|
||||
*/
|
||||
if (backup->status == BACKUP_STATUS_OK &&
|
||||
backup->tli == tli &&
|
||||
(backup->backup_mode == BACKUP_MODE_INCREMENTAL ||
|
||||
(backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
|
||||
backup->backup_mode == BACKUP_MODE_FULL))
|
||||
return backup;
|
||||
}
|
||||
@ -295,7 +295,7 @@ pgBackupCreateDir(pgBackup *backup)
|
||||
void
|
||||
pgBackupWriteConfigSection(FILE *out, pgBackup *backup)
|
||||
{
|
||||
static const char *modes[] = { "", "INCREMENTAL", "FULL"};
|
||||
static const char *modes[] = { "", "PAGE", "FULL"};
|
||||
|
||||
fprintf(out, "# configuration\n");
|
||||
|
||||
@ -488,10 +488,10 @@ parse_backup_mode(const char *value)
|
||||
v++;
|
||||
len = strlen(v);
|
||||
|
||||
if (len > 0 && pg_strncasecmp("full", v, len) == 0)
|
||||
if (len > 0 && pg_strncasecmp("full", v, strlen("full")) == 0)
|
||||
return BACKUP_MODE_FULL;
|
||||
else if (len > 0 && pg_strncasecmp("incremental", v, len) == 0)
|
||||
return BACKUP_MODE_INCREMENTAL;
|
||||
else if (len > 0 && pg_strncasecmp("page", v, strlen("page")) == 0)
|
||||
return BACKUP_MODE_DIFF_PAGE;
|
||||
|
||||
/* Backup mode is invalid, so leave with an error */
|
||||
elog(ERROR_ARGS, _("invalid backup-mode \"%s\""), value);
|
||||
|
||||
4
data.c
4
data.c
@ -526,7 +526,7 @@ restore_data_file(const char *from_root,
|
||||
|
||||
/*
|
||||
* Open backup file for write. We use "r+" at first to overwrite only
|
||||
* modified pages for incremental restore. If the file is not exists,
|
||||
* modified pages for differential restore. If the file is not exists,
|
||||
* re-open it with "w" to create an empty file.
|
||||
*/
|
||||
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
|
||||
@ -652,7 +652,7 @@ restore_data_file(const char *from_root,
|
||||
|
||||
/*
|
||||
* Seek and write the restored page. Backup might have holes in
|
||||
* incremental backups.
|
||||
* differential backups.
|
||||
*/
|
||||
blknum = header.block;
|
||||
if (fseek(out, blknum * BLCKSZ, SEEK_SET) < 0)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# configuration
|
||||
BACKUP_MODE=INCREMENTAL
|
||||
BACKUP_MODE=PAGE
|
||||
COMPRESS_DATA=NO
|
||||
# result
|
||||
TIMELINEID=1
|
||||
|
||||
@ -28,7 +28,7 @@ It proposes the following features:
|
||||
command
|
||||
- Recovery from backup with just one command, with customized targets
|
||||
to facilitate the use of PITR.
|
||||
- Support for full and incremental
|
||||
- Support for full and differential backup
|
||||
- Compression of backup files
|
||||
- Management of backups with integrated catalog
|
||||
|
||||
@ -72,11 +72,11 @@ specify it in PGDATA environmental variable or -D/--pgdata option.
|
||||
Backup target can be one of the following types:
|
||||
|
||||
- Full backup, backup a whole database cluster.
|
||||
- Incremental backup, backup only files or pages modified after the last
|
||||
- Differential backup, backup only files or pages modified after the last
|
||||
verified backup.
|
||||
|
||||
It is recommended to verify backup files as soon as possible after backup.
|
||||
Unverified backup cannot be used in restore and in incremental backup.
|
||||
Unverified backup cannot be used in restore and in differential backup.
|
||||
|
||||
=== RESTORE ===
|
||||
|
||||
@ -140,7 +140,7 @@ Here are some commands to restore from a backup:
|
||||
The fields are:
|
||||
|
||||
* Start: start time of backup
|
||||
* Mode: Mode of backup: FULL (full) or INCR (incremental)
|
||||
* Mode: Mode of backup: FULL (full) or PAGE (page differential)
|
||||
* Current TLI: current timeline of backup
|
||||
* Parent TLI: parent timeline of backup
|
||||
* Time: total time necessary to take this backup
|
||||
@ -214,8 +214,7 @@ absolute paths; relative paths are not allowed.
|
||||
|
||||
*-b* _BACKUPMODE_ / *--backup-mode*=_BACKUPMODE_::
|
||||
Specify backup target files. Available options are: "full",
|
||||
"incremental". Abbreviated forms (prefix match) are also available.
|
||||
For example, -b f means "full" backup.
|
||||
"page".
|
||||
|
||||
*-Z* / *--compress-data*::
|
||||
Compress backup files with zlib if specified.
|
||||
@ -350,7 +349,7 @@ pg_arman has the following restrictions.
|
||||
- If there are some unreadable files/directories in data folder of server
|
||||
WAL directory or archived WAL directory, the backup or restore will fail
|
||||
depending on the backup mode selected.
|
||||
- Incremental backup is not able to take necessary files after a database
|
||||
- Differential backup is not able to take necessary files after a database
|
||||
creation, so take a full backup once a new database is created.
|
||||
|
||||
== DETAILS ==
|
||||
|
||||
@ -5,7 +5,7 @@ CREATE DATABASE
|
||||
0
|
||||
full database backup
|
||||
CHECKPOINT
|
||||
incremental database backup
|
||||
differential database backup
|
||||
CHECKPOINT
|
||||
CHECKPOINT
|
||||
stop DB during running pgbench
|
||||
|
||||
@ -17,7 +17,7 @@ Common Options:
|
||||
-v, --verbose output process information
|
||||
|
||||
Backup options:
|
||||
-b, --backup-mode=MODE full or incremental
|
||||
-b, --backup-mode=MODE full or page
|
||||
-Z, --compress-data compress data backup with zlib
|
||||
-C, --smooth-checkpoint do smooth checkpoint before backup
|
||||
--validate validate backup after taking it
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
Start Mode Current TLI Parent TLI Time Data Status
|
||||
==========================================================================
|
||||
2009-06-03 17:05:53 FULL 1 0 0m ---- RUNNING
|
||||
2009-06-01 17:05:53 INCR 1 0 3m 9223PB DONE
|
||||
2009-06-01 17:05:53 PAGE 1 0 3m 9223PB DONE
|
||||
2009-05-31 17:05:53 FULL 1 0 3m 1242MB DONE
|
||||
\! pg_arman validate -B ${PWD}/results/sample_backup 2009-05-31 17:05:53 --debug
|
||||
INFO: validate: 2009-05-31 17:05:53 backup and archive log files by CRC
|
||||
@ -25,11 +25,11 @@ Start Mode Current TLI Parent TLI Time Data Status
|
||||
==========================================================================
|
||||
2009-06-03 17:05:53 FULL 1 0 0m ---- RUNNING
|
||||
2009-06-02 17:05:03 FULL 1 0 0m ---- DELETED
|
||||
2009-06-01 17:05:53 INCR 1 0 3m 9223PB CORRUPT
|
||||
2009-06-01 17:05:53 PAGE 1 0 3m 9223PB CORRUPT
|
||||
2009-05-31 17:05:53 FULL 1 0 3m 1242MB OK
|
||||
\! pg_arman show 2009-06-01 17:05:53 -A ${PWD}/results/arclog -B ${PWD}/results/sample_backup
|
||||
# configuration
|
||||
BACKUP_MODE=INCREMENTAL
|
||||
BACKUP_MODE=PAGE
|
||||
COMPRESS_DATA=false
|
||||
# result
|
||||
TIMELINEID=1
|
||||
|
||||
@ -217,7 +217,7 @@ pgut_help(bool details)
|
||||
printf(_(" -c, --check show what would have been done\n"));
|
||||
printf(_(" -v, --verbose output process information\n"));
|
||||
printf(_("\nBackup options:\n"));
|
||||
printf(_(" -b, --backup-mode=MODE full or incremental\n"));
|
||||
printf(_(" -b, --backup-mode=MODE full or page\n"));
|
||||
printf(_(" -Z, --compress-data compress data backup with zlib\n"));
|
||||
printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n"));
|
||||
printf(_(" --validate validate backup after taking it\n"));
|
||||
|
||||
@ -110,7 +110,7 @@ typedef enum BackupStatus
|
||||
typedef enum BackupMode
|
||||
{
|
||||
BACKUP_MODE_INVALID,
|
||||
BACKUP_MODE_INCREMENTAL, /* incremental backup */
|
||||
BACKUP_MODE_DIFF_PAGE, /* differential page backup */
|
||||
BACKUP_MODE_FULL /* full backup */
|
||||
} BackupMode;
|
||||
|
||||
@ -140,7 +140,7 @@ typedef struct pgBackup
|
||||
/* Different sizes (-1 means nothing was backed up) */
|
||||
/*
|
||||
* Amount of raw data. For a full backup, this is the total amount of
|
||||
* data while for an incremental backup this is just the differential
|
||||
* data while for a differential backup this is just the difference
|
||||
* of data taken.
|
||||
*/
|
||||
int64 data_bytes;
|
||||
|
||||
@ -174,9 +174,9 @@ base_backup_found:
|
||||
|
||||
last_restored_index = base_index;
|
||||
|
||||
/* restore following incremental backup */
|
||||
/* restore following differential backup */
|
||||
if (verbose)
|
||||
printf(_("searching incremental backup...\n"));
|
||||
printf(_("searching differential backup...\n"));
|
||||
for (i = base_index - 1; i >= 0; i--)
|
||||
{
|
||||
pgBackup *backup = (pgBackup *) parray_get(backups, i);
|
||||
@ -187,7 +187,7 @@ base_backup_found:
|
||||
continue;
|
||||
|
||||
/* use database backup only */
|
||||
if (backup->backup_mode != BACKUP_MODE_INCREMENTAL)
|
||||
if (backup->backup_mode != BACKUP_MODE_DIFF_PAGE)
|
||||
continue;
|
||||
|
||||
/* is the backup is necessary for restore to target timeline ? */
|
||||
|
||||
6
show.c
6
show.c
@ -177,7 +177,7 @@ show_backup_list(FILE *out, parray *backup_list, bool show_all)
|
||||
for (i = 0; i < parray_num(backup_list); i++)
|
||||
{
|
||||
pgBackup *backup;
|
||||
const char *modes[] = { "", "INCR", "FULL"};
|
||||
const char *modes[] = { "", "PAGE", "FULL"};
|
||||
TimeLineID parent_tli;
|
||||
char timestamp[20];
|
||||
char duration[20] = "----";
|
||||
@ -196,8 +196,8 @@ show_backup_list(FILE *out, parray *backup_list, bool show_all)
|
||||
|
||||
/*
|
||||
* Calculate Data field, in the case of full backup this shows the
|
||||
* total amount of data. For an incremental backup, this size is only
|
||||
* the differential of data accumulated.
|
||||
* total amount of data. For an differential backup, this size is only
|
||||
* the difference of data accumulated.
|
||||
*/
|
||||
pretty_size(backup->data_bytes, data_bytes_str,
|
||||
lengthof(data_bytes_str));
|
||||
|
||||
@ -112,10 +112,10 @@ psql --no-psqlrc -p $TEST_PGPORT postgres -c "checkpoint"
|
||||
pg_arman -w -p $TEST_PGPORT backup --verbose -d postgres > $BASE_PATH/results/log_full_1 2>&1
|
||||
|
||||
pgbench -p $TEST_PGPORT -T $DURATION -c 10 pgbench >> $BASE_PATH/results/pgbench.log 2>&1
|
||||
echo "incremental database backup"
|
||||
echo "differential database backup"
|
||||
psql --no-psqlrc -p $TEST_PGPORT postgres -c "checkpoint"
|
||||
#pg_arman -p $TEST_PGPORT backup -b i --verbose -d postgres > $BASE_PATH/results/log_incr1 2>&1
|
||||
pg_arman -w -p $TEST_PGPORT backup -b i --verbose -d postgres > $BASE_PATH/results/log_incr1 2>&1
|
||||
#pg_arman -p $TEST_PGPORT backup -b page --verbose -d postgres > $BASE_PATH/results/log_incr1 2>&1
|
||||
pg_arman -w -p $TEST_PGPORT backup -b page --verbose -d postgres > $BASE_PATH/results/log_incr1 2>&1
|
||||
|
||||
# validate all backup
|
||||
pg_arman validate `date +%Y` --verbose > $BASE_PATH/results/log_validate1 2>&1
|
||||
@ -123,8 +123,8 @@ pg_arman -p $TEST_PGPORT show `date +%Y` -a --verbose -d postgres > $BASE_PATH/r
|
||||
pg_dumpall > $BASE_PATH/results/dump_before_rtx.sql
|
||||
target_xid=`psql --no-psqlrc -p $TEST_PGPORT pgbench -tAq -c "INSERT INTO pgbench_history VALUES (1) RETURNING(xmin);"`
|
||||
psql --no-psqlrc -p $TEST_PGPORT postgres -c "checkpoint"
|
||||
#pg_arman -p $TEST_PGPORT backup -b i --verbose -d postgres > $BASE_PATH/results/log_incr2 2>&1
|
||||
pg_arman -w -p $TEST_PGPORT backup -b i --verbose -d postgres > $BASE_PATH/results/log_incr2 2>&1
|
||||
#pg_arman -p $TEST_PGPORT backup -b page --verbose -d postgres > $BASE_PATH/results/log_incr2 2>&1
|
||||
pg_arman -w -p $TEST_PGPORT backup -b page --verbose -d postgres > $BASE_PATH/results/log_incr2 2>&1
|
||||
|
||||
pgbench -p $TEST_PGPORT -T $DURATION -c 10 pgbench >> $BASE_PATH/results/pgbench.log 2>&1
|
||||
|
||||
@ -185,8 +185,8 @@ diff $BASE_PATH/results/dump_before.sql $BASE_PATH/results/dump_after.sql
|
||||
# incrementa backup can't find last full backup because new timeline started.
|
||||
echo "full database backup after recovery"
|
||||
psql --no-psqlrc -p $TEST_PGPORT postgres -c "checkpoint"
|
||||
#pg_arman -p $TEST_PGPORT backup -b f --verbose -d postgres > $BASE_PATH/results/log_full2 2>&1
|
||||
pg_arman -w -p $TEST_PGPORT backup -b f --verbose -d postgres > $BASE_PATH/results/log_full2 2>&1
|
||||
#pg_arman -p $TEST_PGPORT backup -b full --verbose -d postgres > $BASE_PATH/results/log_full2 2>&1
|
||||
pg_arman -w -p $TEST_PGPORT backup -b full --verbose -d postgres > $BASE_PATH/results/log_full2 2>&1
|
||||
|
||||
# Symbolic links in $ARCLOG_PATH should be deleted.
|
||||
echo "# of symbolic links in ARCLOG_PATH"
|
||||
|
||||
@ -81,7 +81,7 @@ pgBackupValidate(pgBackup *backup,
|
||||
if (!for_get_timeline)
|
||||
{
|
||||
if (backup->backup_mode == BACKUP_MODE_FULL ||
|
||||
backup->backup_mode == BACKUP_MODE_INCREMENTAL)
|
||||
backup->backup_mode == BACKUP_MODE_DIFF_PAGE)
|
||||
elog(INFO, "validate: %s backup and archive log files by %s",
|
||||
timestamp, (size_only ? "SIZE" : "CRC"));
|
||||
}
|
||||
@ -89,7 +89,7 @@ pgBackupValidate(pgBackup *backup,
|
||||
if (!check)
|
||||
{
|
||||
if (backup->backup_mode == BACKUP_MODE_FULL ||
|
||||
backup->backup_mode == BACKUP_MODE_INCREMENTAL)
|
||||
backup->backup_mode == BACKUP_MODE_DIFF_PAGE)
|
||||
{
|
||||
elog(LOG, "database files...");
|
||||
pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR);
|
||||
@ -143,7 +143,7 @@ pgBackupValidateFiles(parray *files, const char *root, bool size_only)
|
||||
if (interrupted)
|
||||
elog(ERROR_INTERRUPTED, _("interrupted during validate"));
|
||||
|
||||
/* skipped backup while incremental backup */
|
||||
/* skipped backup while differential backup */
|
||||
if (file->write_size == BYTES_INVALID || !S_ISREG(file->mode))
|
||||
continue;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user