db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-02-02 13:36:08 +02:00
Code Issues Releases Activity

Add retention show|purge commands. Add tests and documentation.

Browse Source
This commit is contained in:
Arthur Zakirov 2017-02-12 23:42:10 +03:00
parent 7c4c842ce5
commit 976694f1a3
20 changed files with 485 additions and 433 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -25,6 +25,9 @@
/regression.diffs
/regression.out
/results
/env
/tests/__pycache__/
/tests/tmp_dirs/
# Extra files
/datapagemap.c

28
backup.c
View File

@ -58,7 +58,7 @@ typedef struct
*/
static void backup_cleanup(bool fatal, void *userdata);
static void backup_files(void *arg);
static parray *do_backup_database(parray *backup_list, pgBackupOption bkupopt);
static parray *do_backup_database(parray *backup_list, bool smooth_checkpoint);
static void confirm_block_size(const char *name, int blcksz);
static void pg_start_backup(const char *label, bool smooth, pgBackup *backup);
static void pg_stop_backup(pgBackup *backup);
@ -96,7 +96,7 @@ static void StreamLog(void *arg);
* Take a backup of database and return the list of files backed up.
*/
static parray *
do_backup_database(parray *backup_list, pgBackupOption bkupopt)
do_backup_database(parray *backup_list, bool smooth_checkpoint)
{
int i;
parray *prev_files = NULL; /* file list of previous database backup */
@ -113,9 +113,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
backup_files_args *backup_threads_args[num_threads];
bool is_ptrack_support;
/* repack the options */
bool smooth_checkpoint = bkupopt.smooth_checkpoint;
pgBackup *prev_backup = NULL;
/* Block backup operations on a standby */
@ -446,15 +444,11 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
int
do_backup(pgBackupOption bkupopt)
do_backup(bool smooth_checkpoint)
{
parray *backup_list;
parray *files_database;
int ret;
/* repack the necessary options */
int keep_data_generations = bkupopt.keep_data_generations;
int keep_data_days = bkupopt.keep_data_days;
int ret;
parray *backup_list;
parray *files_database;
/* PGDATA and BACKUP_MODE are always required */
if (pgdata == NULL)
@ -481,12 +475,12 @@ do_backup(pgBackupOption bkupopt)
elog(LOG, "----------------------------------------");
/* get exclusive lock of backup catalog */
ret = catalog_lock();
ret = catalog_lock(true);
if (ret == -1)
elog(ERROR, "cannot lock backup catalog");
else if (ret == 1)
elog(ERROR,
"another pg_probackup is running, skipping this backup");
"another pg_probackup is running, skipping this backup");
/* initialize backup result */
current.status = BACKUP_STATUS_RUNNING;
@ -521,7 +515,7 @@ do_backup(pgBackupOption bkupopt)
pgut_atexit_push(backup_cleanup, NULL);
/* backup data */
files_database = do_backup_database(backup_list, bkupopt);
files_database = do_backup_database(backup_list, smooth_checkpoint);
pgut_atexit_pop(backup_cleanup, NULL);
/* update backup status to DONE */
@ -550,10 +544,6 @@ do_backup(pgBackupOption bkupopt)
elog(LOG, "========================================");
}
/* Delete old backup files after all backup operation. */
pgBackupDelete(keep_data_generations, keep_data_days);
/* Cleanup backup mode file list */
if (files_database)
parray_walk(files_database, pgFileFree);

74
catalog.c
View File

@ -21,7 +21,7 @@
#include "pgut/pgut-port.h"
static pgBackup *catalog_read_ini(const char *path);
static pgBackup *read_backup_from_file(const char *path);
#define BOOL_TO_STR(val) ((val) ? "true" : "false")
@ -32,17 +32,16 @@ static int lock_fd = -1;
* If the lock is held by another one, return 1 immediately.
*/
int
catalog_lock(void)
catalog_lock(bool check_catalog)
{
int ret;
char id_path[MAXPGPATH];
int ret;
char id_path[MAXPGPATH];
join_path_components(id_path, backup_path, PG_RMAN_INI_FILE);
join_path_components(id_path, backup_path, BACKUP_CATALOG_CONF_FILE);
lock_fd = open(id_path, O_RDWR);
if (lock_fd == -1)
elog(errno == ENOENT ? ERROR : ERROR,
"cannot open file \"%s\": %s", id_path, strerror(errno));
#ifdef __IBMC__
ret = lockf(lock_fd, LOCK_EX | LOCK_NB, 0); /* non-blocking */
#else
@ -64,6 +63,19 @@ catalog_lock(void)
}
}
if (check_catalog)
{
uint64 id;
Assert(pgdata);
/* Check system-identifier */
id = get_system_identifier(true);
if (id != system_identifier)
elog(ERROR, "Backup directory was initialized for system id = %ld, but target system id = %ld",
system_identifier, id);
}
return 0;
}
@ -82,15 +94,15 @@ catalog_unlock(void)
* If no backup matches, return NULL.
*/
pgBackup *
catalog_get_backup(time_t timestamp)
read_backup(time_t timestamp)
{
pgBackup tmp;
char ini_path[MAXPGPATH];
char conf_path[MAXPGPATH];
tmp.start_time = timestamp;
pgBackupGetPath(&tmp, ini_path, lengthof(ini_path), BACKUP_INI_FILE);
pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONF_FILE);
return catalog_read_ini(ini_path);
return read_backup_from_file(conf_path);
}
static bool
@ -144,8 +156,8 @@ catalog_get_backup_list(time_t backup_id)
join_path_components(date_path, backups_path, date_ent->d_name);
/* read backup information from backup.ini */
snprintf(ini_path, MAXPGPATH, "%s/%s", date_path, BACKUP_INI_FILE);
backup = catalog_read_ini(ini_path);
snprintf(ini_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONF_FILE);
backup = read_backup_from_file(ini_path);
/* ignore corrupted backup */
if (backup)
@ -309,7 +321,7 @@ pgBackupWriteIni(pgBackup *backup)
FILE *fp = NULL;
char ini_path[MAXPGPATH];
pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_INI_FILE);
pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_CONF_FILE);
fp = fopen(ini_path, "wt");
if (fp == NULL)
elog(ERROR, "cannot open INI file \"%s\": %s", ini_path,
@ -330,7 +342,7 @@ pgBackupWriteIni(pgBackup *backup)
* - Do not care section.
*/
static pgBackup *
catalog_read_ini(const char *path)
read_backup_from_file(const char *path)
{
pgBackup *backup;
char *backup_mode = NULL;
@ -342,21 +354,21 @@ catalog_read_ini(const char *path)
pgut_option options[] =
{
{'s', 0, "backup-mode", NULL, SOURCE_ENV},
{'u', 0, "timelineid", NULL, SOURCE_ENV},
{'s', 0, "start-lsn", NULL, SOURCE_ENV},
{'s', 0, "stop-lsn", NULL, SOURCE_ENV},
{'t', 0, "start-time", NULL, SOURCE_ENV},
{'t', 0, "end-time", NULL, SOURCE_ENV},
{'U', 0, "recovery-xid", NULL, SOURCE_ENV},
{'t', 0, "recovery-time", NULL, SOURCE_ENV},
{'I', 0, "data-bytes", NULL, SOURCE_ENV},
{'u', 0, "block-size", NULL, SOURCE_ENV},
{'u', 0, "xlog-block-size", NULL, SOURCE_ENV},
{'u', 0, "checksum_version", NULL, SOURCE_ENV},
{'u', 0, "stream", NULL, SOURCE_ENV},
{'s', 0, "status", NULL, SOURCE_ENV},
{'s', 0, "parent_backup", NULL, SOURCE_ENV},
{'s', 0, "backup-mode", NULL, SOURCE_FILE_STRICT},
{'u', 0, "timelineid", NULL, SOURCE_FILE_STRICT},
{'s', 0, "start-lsn", NULL, SOURCE_FILE_STRICT},
{'s', 0, "stop-lsn", NULL, SOURCE_FILE_STRICT},
{'t', 0, "start-time", NULL, SOURCE_FILE_STRICT},
{'t', 0, "end-time", NULL, SOURCE_FILE_STRICT},
{'U', 0, "recovery-xid", NULL, SOURCE_FILE_STRICT},
{'t', 0, "recovery-time", NULL, SOURCE_FILE_STRICT},
{'I', 0, "data-bytes", NULL, SOURCE_FILE_STRICT},
{'u', 0, "block-size", NULL, SOURCE_FILE_STRICT},
{'u', 0, "xlog-block-size", NULL, SOURCE_FILE_STRICT},
{'u', 0, "checksum_version", NULL, SOURCE_FILE_STRICT},
{'u', 0, "stream", NULL, SOURCE_FILE_STRICT},
{'s', 0, "status", NULL, SOURCE_FILE_STRICT},
{'s', 0, "parent_backup", NULL, SOURCE_FILE_STRICT},
{0}
};
@ -364,7 +376,7 @@ catalog_read_ini(const char *path)
return NULL;
backup = pgut_new(pgBackup);
catalog_init_config(backup);
init_backup(backup);
i = 0;
options[i++].var = &backup_mode;
@ -516,7 +528,7 @@ pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subd
}
void
catalog_init_config(pgBackup *backup)
init_backup(pgBackup *backup)
{
backup->backup_mode = BACKUP_MODE_INVALID;
backup->status = BACKUP_STATUS_INVALID;

352
delete.c
View File

@ -10,10 +10,12 @@
#include "pg_probackup.h"
#include <dirent.h>
#include <time.h>
#include <unistd.h>
static int pgBackupDeleteFiles(pgBackup *backup);
int do_deletewal(time_t backup_id, bool strict);
static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
bool delete_all);
int
do_delete(time_t backup_id)
@ -29,12 +31,12 @@ do_delete(time_t backup_id)
elog(ERROR, "required backup ID not specified");
/* Lock backup catalog */
ret = catalog_lock();
ret = catalog_lock(false);
if (ret == -1)
elog(ERROR, "can't lock backup catalog.");
else if (ret == 1)
elog(ERROR,
"another pg_probackup is running, stop delete.");
"another pg_probackup is running, stop delete.");
/* Get complete list of backups */
backup_list = catalog_get_backup_list(0);
@ -88,37 +90,40 @@ found_backup:
return 0;
}
int do_deletewal(time_t backup_id, bool strict)
/*
* Delete in archive WAL segments that are not needed anymore. The oldest
* segment to be kept is the first segment that the oldest full backup
* found around needs to keep.
*/
int
do_deletewal(time_t backup_id, bool strict)
{
int i;
size_t i;
int ret;
parray *backup_list;
XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
TimeLineID oldest_tli;
pgBackup *last_backup;
bool backup_found = false;
/*
* Delete in archive WAL segments that are not needed anymore. The oldest
* segment to be kept is the first segment that the oldest full backup
* found around needs to keep.
*/
/* Lock backup catalog */
ret = catalog_lock();
ret = catalog_lock(false);
if (ret == -1)
elog(ERROR, "can't lock backup catalog.");
else if (ret == 1)
elog(ERROR,
"another pg_probackup is running, stop delete.");
"another pg_probackup is running, stop delete.");
/* Find oldest LSN, used by backups */
backup_list = catalog_get_backup_list(0);
for (i = 0; i < parray_num(backup_list); i++)
{
last_backup = (pgBackup *) parray_get(backup_list, i);
pgBackup *last_backup = (pgBackup *) parray_get(backup_list, i);
if (last_backup->status == BACKUP_STATUS_OK)
{
oldest_lsn = last_backup->start_lsn;
oldest_tli = last_backup->tli;
if (strict && backup_id != 0 && backup_id >= last_backup->start_time)
{
backup_found = true;
@ -126,196 +131,98 @@ int do_deletewal(time_t backup_id, bool strict)
}
}
}
if (strict && backup_id != 0 && backup_found == false)
elog(ERROR, "not found backup for deletwal command");
catalog_unlock();
parray_walk(backup_list, pgBackupFree);
parray_free(backup_list);
if (!XLogRecPtrIsInvalid(oldest_lsn))
{
XLogSegNo targetSegNo;
char oldestSegmentNeeded[MAXFNAMELEN];
DIR *arcdir;
struct dirent *arcde;
char wal_file[MAXPGPATH];
char max_wal_file[MAXPGPATH];
char min_wal_file[MAXPGPATH];
int rc;
max_wal_file[0] = '\0';
min_wal_file[0] = '\0';
XLByteToSeg(oldest_lsn, targetSegNo);
XLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo);
elog(LOG, "Removing segments older than %s", oldestSegmentNeeded);
/*
* Now is time to do the actual work and to remove all the segments
* not needed anymore.
*/
if ((arcdir = opendir(arclog_path)) != NULL)
{
while (errno = 0, (arcde = readdir(arcdir)) != NULL)
{
/*
* We ignore the timeline part of the XLOG segment identifiers in
* deciding whether a segment is still needed. This ensures that
* we won't prematurely remove a segment from a parent timeline.
* We could probably be a little more proactive about removing
* segments of non-parent timelines, but that would be a whole lot
* more complicated.
*
* We use the alphanumeric sorting property of the filenames to
* decide which ones are earlier than the exclusiveCleanupFileName
* file. Note that this means files are not removed in the order
* they were originally written, in case this worries you.
*/
if ((IsXLogFileName(arcde->d_name) ||
IsPartialXLogFileName(arcde->d_name)) &&
strcmp(arcde->d_name + 8, oldestSegmentNeeded + 8) < 0)
{
/*
* Use the original file name again now, including any
* extension that might have been chopped off before testing
* the sequence.
*/
snprintf(wal_file, MAXPGPATH, "%s/%s",
arclog_path, arcde->d_name);
rc = unlink(wal_file);
if (rc != 0)
{
elog(WARNING, "could not remove file \"%s\": %s",
wal_file, strerror(errno));
break;
}
if (verbose)
elog(LOG, "removed WAL segment \"%s\"", wal_file);
if (max_wal_file[0] == '\0' ||
strcmp(max_wal_file + 8, arcde->d_name + 8) < 0)
{
strcpy(max_wal_file, arcde->d_name);
}
if (min_wal_file[0] == '\0' ||
strcmp(min_wal_file + 8, arcde->d_name + 8) > 0)
{
strcpy(min_wal_file, arcde->d_name);
}
}
}
if (!verbose && min_wal_file[0] != '\0')
elog(NOTICE, "removed min WAL segment \"%s\"", min_wal_file);
if (!verbose && max_wal_file[0] != '\0')
elog(NOTICE, "removed max WAL segment \"%s\"", max_wal_file);
if (errno)
elog(WARNING, "could not read archive location \"%s\": %s",
arclog_path, strerror(errno));
if (closedir(arcdir))
elog(WARNING, "could not close archive location \"%s\": %s",
arclog_path, strerror(errno));
}
else
elog(WARNING, "could not open archive location \"%s\": %s",
arclog_path, strerror(errno));
}
delete_walfiles(oldest_lsn, oldest_tli, false);
return 0;
}
/*
* Delete backups that are older than KEEP_xxx_DAYS and have more generations
* than KEEP_xxx_FILES.
* Remove backups by retention policy. Retention policy is configured by
* retention_redundancy and retention_window variables.
*/
void
pgBackupDelete(int keep_generations, int keep_days)
int
do_retention_purge(void)
{
int i;
parray *backup_list;
int backup_num;
time_t days_threshold = current.start_time - (keep_days * 60 * 60 * 24);
parray *backup_list;
uint32 backup_num;
size_t i;
time_t days_threshold = time(NULL) - (retention_window * 60 * 60 * 24);
XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
TimeLineID oldest_tli;
int ret;
if (verbose)
{
char generations_str[100];
char days_str[100];
if (retention_redundancy > 0)
elog(LOG, "REDUNDANCY=%u", retention_redundancy);
if (retention_window > 0)
elog(LOG, "WINDOW=%u", retention_window);
if (keep_generations == KEEP_INFINITE)
strncpy(generations_str, "INFINITE",
lengthof(generations_str));
else
snprintf(generations_str, lengthof(generations_str),
"%d", keep_generations);
if (retention_redundancy == 0 && retention_window == 0)
elog(ERROR, "retention policy is not set");
if (keep_days == KEEP_INFINITE)
strncpy(days_str, "INFINITE", lengthof(days_str));
else
snprintf(days_str, lengthof(days_str), "%d", keep_days);
elog(LOG, "deleted old backups (generations=%s, days=%s)",
generations_str, days_str);
}
/* Leave if an infinite generation of backups is kept */
if (keep_generations == KEEP_INFINITE && keep_days == KEEP_INFINITE)
{
elog(LOG, "%s() infinite", __FUNCTION__);
return;
}
/* Lock backup catalog */
ret = catalog_lock(false);
if (ret == 1)
elog(ERROR,
"cannot lock backup catalog, another pg_probackup is running");
/* Get a complete list of backups. */
backup_list = catalog_get_backup_list(0);
if (parray_num(backup_list) == 0)
{
elog(INFO, "backup list is empty");
elog(INFO, "exit");
catalog_unlock();
return 0;
}
/* Find target backups to be deleted */
backup_num = 0;
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
int backup_num_evaluate = backup_num;
elog(LOG, "%s() %lu", __FUNCTION__, backup->start_time);
uint32 backup_num_evaluate = backup_num;
/*
* When a validate full backup was found, we can delete the
* backup that is older than it using the number of generations.
*/
if (backup->backup_mode == BACKUP_MODE_FULL &&
backup->status == BACKUP_STATUS_OK)
if (backup->backup_mode == BACKUP_MODE_FULL)
backup_num++;
/* Evaluate if this backup is eligible for removal */
if (backup_num_evaluate + 1 <= keep_generations &&
keep_generations != KEEP_INFINITE)
if (backup_num_evaluate + 1 <= retention_redundancy ||
(retention_window > 0 && backup->start_time >= days_threshold))
{
/* Do not include the latest full backup in this count */
elog(LOG, "%s() backup are only %d", __FUNCTION__, backup_num);
continue;
}
else if (backup->start_time >= days_threshold &&
keep_days != KEEP_INFINITE)
{
/*
* If the start time of the backup is older than the threshold and
* there are enough generations of full backups, delete the backup.
*/
elog(LOG, "%s() %lu is not older than %lu", __FUNCTION__,
backup->start_time, days_threshold);
/* Save LSN and Timeline to remove unnecessary WAL segments */
oldest_lsn = backup->start_lsn;
oldest_tli = backup->tli;
continue;
}
elog(LOG, "%s() %lu is older than %lu", __FUNCTION__,
backup->start_time, days_threshold);
/* delete backup and update status to DELETED */
/* Delete backup and update status to DELETED */
pgBackupDeleteFiles(backup);
}
/* cleanup */
/* Purge WAL files */
delete_walfiles(oldest_lsn, oldest_tli, true);
/* Cleanup */
parray_walk(backup_list, pgBackupFree);
parray_free(backup_list);
catalog_unlock();
elog(INFO, "purging is finished");
return 0;
}
/*
@ -325,10 +232,10 @@ pgBackupDelete(int keep_generations, int keep_days)
static int
pgBackupDeleteFiles(pgBackup *backup)
{
int i;
char path[MAXPGPATH];
char timestamp[20];
parray *files;
size_t i;
char path[MAXPGPATH];
char timestamp[20];
parray *files;
/*
* If the backup was deleted already, there is nothing to do.
@ -340,7 +247,6 @@ pgBackupDeleteFiles(pgBackup *backup)
elog(INFO, "delete: %s %s", base36enc(backup->start_time), timestamp);
/*
* Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which
* the error occurs before deleting all backup files.
@ -363,7 +269,7 @@ pgBackupDeleteFiles(pgBackup *backup)
pgFile *file = (pgFile *) parray_get(files, i);
/* print progress */
elog(LOG, "delete file(%d/%lu) \"%s\"", i + 1,
elog(LOG, "delete file(%zd/%lu) \"%s\"", i + 1,
(unsigned long) parray_num(files), file->path);
/* skip actual deletion in check mode */
@ -375,6 +281,7 @@ pgBackupDeleteFiles(pgBackup *backup)
strerror(errno));
parray_walk(files, pgFileFree);
parray_free(files);
return 1;
}
}
@ -385,3 +292,112 @@ pgBackupDeleteFiles(pgBackup *backup)
return 0;
}
/*
* Delete WAL segments up to oldest_lsn.
*
* If oldest_lsn is invalid function exists. But if delete_all is true then
* WAL segements will be deleted anyway.
*/
static void
delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all)
{
XLogSegNo targetSegNo;
char oldestSegmentNeeded[MAXFNAMELEN];
DIR *arcdir;
struct dirent *arcde;
char wal_file[MAXPGPATH];
char max_wal_file[MAXPGPATH];
char min_wal_file[MAXPGPATH];
int rc;
if (XLogRecPtrIsInvalid(oldest_lsn) && !delete_all)
return;
max_wal_file[0] = '\0';
min_wal_file[0] = '\0';
if (!XLogRecPtrIsInvalid(oldest_lsn))
{
XLByteToSeg(oldest_lsn, targetSegNo);
XLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo);
elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded);
}
else
elog(LOG, "removing all WAL segments");
/*
* Now it is time to do the actual work and to remove all the segments
* not needed anymore.
*/
if ((arcdir = opendir(arclog_path)) != NULL)
{
while (errno = 0, (arcde = readdir(arcdir)) != NULL)
{
/*
* We ignore the timeline part of the XLOG segment identifiers in
* deciding whether a segment is still needed. This ensures that
* we won't prematurely remove a segment from a parent timeline.
* We could probably be a little more proactive about removing
* segments of non-parent timelines, but that would be a whole lot
* more complicated.
*
* We use the alphanumeric sorting property of the filenames to
* decide which ones are earlier than the exclusiveCleanupFileName
* file. Note that this means files are not removed in the order
* they were originally written, in case this worries you.
*/
if (IsXLogFileName(arcde->d_name) ||
IsPartialXLogFileName(arcde->d_name) ||
IsBackupHistoryFileName(arcde->d_name))
{
if (XLogRecPtrIsInvalid(oldest_lsn) ||
strncmp(arcde->d_name, oldestSegmentNeeded,
XLOG_FNAME_LEN) < 0)
{
/*
* Use the original file name again now, including any
* extension that might have been chopped off before testing
* the sequence.
*/
snprintf(wal_file, MAXPGPATH, "%s/%s",
arclog_path, arcde->d_name);
rc = unlink(wal_file);
if (rc != 0)
{
elog(WARNING, "could not remove file \"%s\": %s",
wal_file, strerror(errno));
break;
}
if (verbose)
elog(LOG, "removed WAL segment \"%s\"", wal_file);
if (max_wal_file[0] == '\0' ||
strcmp(max_wal_file + 8, arcde->d_name + 8) < 0)
strcpy(max_wal_file, arcde->d_name);
if (min_wal_file[0] == '\0' ||
strcmp(min_wal_file + 8, arcde->d_name + 8) > 0)
strcpy(min_wal_file, arcde->d_name);
}
}
}
if (!verbose && min_wal_file[0] != '\0')
elog(INFO, "removed min WAL segment \"%s\"", min_wal_file);
if (!verbose && max_wal_file[0] != '\0')
elog(INFO, "removed max WAL segment \"%s\"", max_wal_file);
if (errno)
elog(WARNING, "could not read archive location \"%s\": %s",
arclog_path, strerror(errno));
if (closedir(arcdir))
elog(WARNING, "could not close archive location \"%s\": %s",
arclog_path, strerror(errno));
}
else
elog(WARNING, "could not open archive location \"%s\": %s",
arclog_path, strerror(errno));
}

49
doc/pg_probackup.md
View File

@ -11,6 +11,7 @@ pg_probackup [option...] validate backup_ID
pg_probackup [option...] show [backup_ID]
pg_probackup [option...] delete backup_ID
pg_probackup [option...] delwal [backup_ID]
pg_probackup [option...] retention show|purge
```
## Description
@ -294,6 +295,34 @@ The same backup directory can be used for pg\_probackup on both servers, primary
A backup can be used to restore primary database server as well as standby. It depends on the server on which pg\_probackup is executed with restore command. Note that recovered PostgreSQL will always run as primary server if started right after the pg\_probackup. To run it as standby, edit recovery.conf file created by pg\_probackup: at least delete every parameter that specify recovery target (recovery\_target, recovery\_target\_time, and recovery\_target\_xid), change target timeline to 'latest', and add standby\_mode = 'on'. Probably primary\_conninfo should be added too for streaming replication, and hot\_standby = 'on' in database configuration parameters for hot standby mode.
### Backup Retention Policy
It is possible to configure the backup retention policy. The retention policy
specifies specifies which backups must be kept to meet data recoverability
requirements. The policy can be configured using two parameters: redundancy and
window.
Redundancy parameter specifies how many full backups purge command should keep.
For example, you make a full backup on Sunday and an incremental backup every day.
If redundancy is 1, then this backup will become obsolete on next Sunday and will
be deleted with all its incremental backups when next full backup will be created.
Window parameter specifies the number of days of data recoverability. If window
is 14, then all full backups older than 14 days will be deleted with their
incremental backups.
This parameters can be used together. Backups are obsolete if they don't
meet both parameters. For example, you have retention is 1, window is 14 and
two full backups are made 3 and 7 days ago. In this situation both backups aren't
obsolete and will be kept 14 days.
To delete obsolete backups execute the following command:
```
pg_probackup retention purge
```
Redundancy and window parameters values will be taken from command line options
or from pg_probackup.conf configuration file.
## Additional Features
### Parallel Execution
@ -319,11 +348,11 @@ Pages are packed before going to backup, leaving unused parts of pages behind (s
Whether page checksums are enabled or not, pg\_probackup calculates checksums for each file in a backup. Checksums are checked immediately after backup is taken and right before restore, to timely detect possible backup corruptions.
##Options
## Options
Options for pg\_probackup utility can be specified in command line (such options are shown below starting from either one or two minus signs). If not given in command line, values for some options are derived from environmental variables (names of environmental variables are in uppercase). Otherwise values for some options are taken from pg\_probackup.conf configuration file, located in the backup directory (such option names are in lowercase).
Common options:
### Common options:
-B _directory_
--backup-path=_directory_
@ -365,7 +394,7 @@ Show quick help on command line options.
Show version information.
Backup options:
### Backup options:
-b _mode_
--backup-mode=_mode_
@ -430,7 +459,7 @@ Never issue a password prompt. If the server requires password authentication an
Force pg\_probackup to prompt for a password before connecting to a database.
Restore options:
### Restore options:
--time
@ -448,12 +477,22 @@ Specifies whether to stop just after the specified recovery target (true), or ju
Specifies recovering into a particular timeline.
Delete options:
### Delete options:
--wal
Delete WAL files that are no longer necessary to restore from any of existing backups.
### Retention policy options:
--redundancy
Specifies how many full backups purge command should keep.
--window
Specifies the number of days of recoverability.
## Restrictions
Currently pg\_probackup has the following restrictions:

2
fetch.c
View File

@ -3,7 +3,7 @@
* fetch.c
* Functions for fetching files from PostgreSQL data directory
*
* Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
* Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
*
*-------------------------------------------------------------------------
*/

40
init.c
View File

@ -12,9 +12,6 @@
#include <unistd.h>
#include <dirent.h>
static void parse_postgresql_conf(const char *path, char **log_directory,
char **archive_command);
/*
* selects function for scandir.
*/
@ -31,8 +28,6 @@ do_init(void)
{
char path[MAXPGPATH];
char arclog_path_dir[MAXPGPATH];
char *log_directory = NULL;
char *archive_command = NULL;
FILE *fp;
uint64 _system_identifier;
@ -58,47 +53,20 @@ do_init(void)
join_path_components(path, backup_path, BACKUPS_DIR);
dir_create_dir(path, DIR_PERMISSION);
/* read postgresql.conf */
if (pgdata)
{
join_path_components(path, pgdata, "postgresql.conf");
parse_postgresql_conf(path, &log_directory, &archive_command);
}
/* Create "wal" directory */
join_path_components(arclog_path_dir, backup_path, "wal");
dir_create_dir(arclog_path_dir, DIR_PERMISSION);
_system_identifier = get_system_identifier(false);
/* create pg_probackup.conf */
join_path_components(path, backup_path, PG_RMAN_INI_FILE);
join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE);
fp = fopen(path, "wt");
if (fp == NULL)
elog(ERROR, "cannot create pg_probackup.conf: %s", strerror(errno));
join_path_components(arclog_path_dir, backup_path, "wal");
dir_create_dir(arclog_path_dir, DIR_PERMISSION);
fprintf(fp, "system-identifier = %li\n", _system_identifier);
fprintf(fp, "\n");
fclose(fp);
free(archive_command);
free(log_directory);
return 0;
}
static void
parse_postgresql_conf(const char *path,
char **log_directory,
char **archive_command)
{
pgut_option options[] =
{
{ 's', 0, "log_directory" , NULL, SOURCE_ENV },
{ 's', 0, "archive_command" , NULL, SOURCE_ENV },
{ 0 }
};
options[0].var = log_directory;
options[1].var = archive_command;
pgut_readopt(path, options, LOG); /* ignore unknown options */
}

120
pg_probackup.c
View File

@ -32,15 +32,12 @@ pgBackup current;
/* backup configuration */
static bool smooth_checkpoint;
static int keep_data_generations = KEEP_INFINITE;
static int keep_data_days = KEEP_INFINITE;
int num_threads = 1;
bool stream_wal = false;
bool from_replica = false;
static bool backup_logs = false;
bool progress = false;
bool delete_wal = false;
uint64 system_identifier = 0;
/* restore configuration */
static char *target_time;
@ -48,26 +45,30 @@ static char *target_xid;
static char *target_inclusive;
static TimeLineID target_tli;
uint64 system_identifier = 0;
/* retention configuration */
uint32 retention_redundancy = 0;
uint32 retention_window = 0;
static void opt_backup_mode(pgut_option *opt, const char *arg);
static pgut_option options[] =
{
/* directory options */
{ 's', 'D', "pgdata", &pgdata, SOURCE_ENV },
{ 's', 'B', "backup-path", &backup_path, SOURCE_ENV },
{ 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE },
{ 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE },
/* common options */
/* { 'b', 'c', "check", &check },*/
{ 'i', 'j', "threads", &num_threads },
{ 'b', 8, "stream", &stream_wal },
{ 'b', 11, "progress", &progress },
{ 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE },
{ 'b', 8, "stream", &stream_wal, SOURCE_CMDLINE },
{ 'b', 11, "progress", &progress, SOURCE_CMDLINE },
/* backup options */
{ 'b', 10, "backup-pg-log", &backup_logs },
{ 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_ENV },
{ 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_ENV },
{ 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE },
{ 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE },
{ 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE },
{ 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE },
/* options with only long name (keep-xxx) */
/* { 'i', 1, "keep-data-generations", &keep_data_generations, SOURCE_ENV },
{ 'i', 2, "keep-data-days", &keep_data_days, SOURCE_ENV },*/
/* restore options */
{ 's', 3, "time", &target_time, SOURCE_CMDLINE },
{ 's', 4, "xid", &target_xid, SOURCE_CMDLINE },
@ -75,8 +76,11 @@ static pgut_option options[] =
{ 'u', 6, "timeline", &target_tli, SOURCE_CMDLINE },
/* delete options */
{ 'b', 12, "wal", &delete_wal },
/* retention options */
{ 'u', 13, "redundancy", &retention_redundancy, SOURCE_CMDLINE },
{ 'u', 14, "window", &retention_window, SOURCE_CMDLINE },
/* other */
{ 'U', 13, "system-identifier", &system_identifier, SOURCE_FILE },
{ 'U', 15, "system-identifier", &system_identifier, SOURCE_FILE_STRICT },
{ 0 }
};
@ -86,7 +90,8 @@ static pgut_option options[] =
int
main(int argc, char *argv[])
{
const char *cmd = NULL;
const char *cmd = NULL,
*subcmd = NULL;
const char *backup_id_string = NULL;
time_t backup_id = 0;
int i;
@ -95,7 +100,7 @@ main(int argc, char *argv[])
setvbuf(stdout, 0, _IONBF, 0); /* TODO: remove this */
/* initialize configuration */
catalog_init_config(&current);
init_backup(&current);
/* overwrite configuration with command line arguments */
i = pgut_getopt(argc, argv, options);
@ -103,15 +108,15 @@ main(int argc, char *argv[])
for (; i < argc; i++)
{
if (cmd == NULL)
{
cmd = argv[i];
if(strcmp(cmd, "show") != 0 &&
strcmp(cmd, "validate") != 0 &&
strcmp(cmd, "delete") != 0 &&
strcmp(cmd, "restore") != 0 &&
strcmp(cmd, "delwal") != 0)
break;
} else if (backup_id_string == NULL)
else if (strcmp(cmd, "retention") == 0)
subcmd = argv[i];
else if (backup_id_string == NULL &&
(strcmp(cmd, "show") == 0 ||
strcmp(cmd, "validate") == 0 ||
strcmp(cmd, "delete") == 0 ||
strcmp(cmd, "restore") == 0 ||
strcmp(cmd, "delwal") == 0))
backup_id_string = argv[i];
else
elog(ERROR, "too many arguments");
@ -132,38 +137,36 @@ main(int argc, char *argv[])
}
}
/* Read default configuration from file. */
if (backup_path)
/* BACKUP_PATH is always required */
if (backup_path == NULL)
elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)");
else
{
char path[MAXPGPATH];
char path[MAXPGPATH];
/* Check if backup_path is directory. */
struct stat stat_buf;
int rc = stat(backup_path, &stat_buf);
int rc = stat(backup_path, &stat_buf);
/* If rc == -1, there is no file or directory. So it's OK. */
if (rc != -1 && !S_ISDIR(stat_buf.st_mode))
elog(ERROR, "-B, --backup-path must be a path to directory");
join_path_components(path, backup_path, PG_RMAN_INI_FILE);
join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE);
pgut_readopt(path, options, ERROR);
/* setup stream options */
if (pgut_dbname != NULL)
dbname = pstrdup(pgut_dbname);
if (host != NULL)
dbhost = pstrdup(host);
if (port != NULL)
dbport = pstrdup(port);
if (username != NULL)
dbuser = pstrdup(username);
}
/* BACKUP_PATH is always required */
if (backup_path == NULL)
elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)");
/* setup stream options */
if (pgut_dbname != NULL)
dbname = pstrdup(pgut_dbname);
if (host != NULL)
dbhost = pstrdup(host);
if (port != NULL)
dbport = pstrdup(port);
if (username != NULL)
dbuser = pstrdup(username);
/* path must be absolute */
if (backup_path != NULL && !is_absolute_path(backup_path))
if (!is_absolute_path(backup_path))
elog(ERROR, "-B, --backup-path must be an absolute path");
if (pgdata != NULL && !is_absolute_path(pgdata))
elog(ERROR, "-D, --pgdata must be an absolute path");
@ -189,20 +192,10 @@ main(int argc, char *argv[])
return do_init();
else if (pg_strcasecmp(cmd, "backup") == 0)
{
pgBackupOption bkupopt;
int res;
uint64 _system_identifier;
bkupopt.smooth_checkpoint = smooth_checkpoint;
bkupopt.keep_data_generations = keep_data_generations;
bkupopt.keep_data_days = keep_data_days;
_system_identifier = get_system_identifier(true);
if (_system_identifier != system_identifier)
elog(ERROR, "Backup directory was initialized for system id = %ld, but target system id = %ld",
system_identifier, _system_identifier);
int res;
/* Do the backup */
res = do_backup(bkupopt);
res = do_backup(smooth_checkpoint);
if (res != 0)
return res;
@ -230,6 +223,15 @@ main(int argc, char *argv[])
return do_delete(backup_id);
else if (pg_strcasecmp(cmd, "delwal") == 0)
return do_deletewal(backup_id, true);
else if (pg_strcasecmp(cmd, "retention") == 0)
{
if (subcmd == NULL)
elog(ERROR, "you must specify retention command");
else if (pg_strcasecmp(subcmd, "show") == 0)
return do_retention_show();
else if (pg_strcasecmp(subcmd, "purge") == 0)
return do_retention_purge();
}
else
elog(ERROR, "invalid command \"%s\"", cmd);
@ -243,11 +245,12 @@ pgut_help(bool details)
printf(_("Usage:\n"));
printf(_(" %s [option...] init\n"), PROGRAM_NAME);
printf(_(" %s [option...] backup\n"), PROGRAM_NAME);
printf(_(" %s [option...] restore\n"), PROGRAM_NAME);
printf(_(" %s [option...] restore [backup-ID]\n"), PROGRAM_NAME);
printf(_(" %s [option...] show [backup-ID]\n"), PROGRAM_NAME);
printf(_(" %s [option...] validate backup-ID\n"), PROGRAM_NAME);
printf(_(" %s [option...] delete backup-ID\n"), PROGRAM_NAME);
printf(_(" %s [option...] delwal [backup-ID]\n"), PROGRAM_NAME);
printf(_(" %s [option...] retention show|purge\n"), PROGRAM_NAME);
if (!details)
return;
@ -260,8 +263,6 @@ pgut_help(bool details)
printf(_(" -b, --backup-mode=MODE backup mode (full, page, ptrack)\n"));
printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n"));
printf(_(" --stream stream the transaction log and include it in the backup\n"));
/*printf(_(" --keep-data-generations=N keep GENERATION of full data backup\n"));
printf(_(" --keep-data-days=DAY keep enough data backup to recover to DAY days age\n"));*/
printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
printf(_(" --backup-pg-log backup of pg_log directory\n"));
printf(_(" -j, --threads=NUM number of parallel threads\n"));
@ -275,6 +276,9 @@ pgut_help(bool details)
printf(_(" --progress show progress\n"));
printf(_("\nDelete options:\n"));
printf(_(" --wal remove unnecessary wal files\n"));
printf(_("\nRetention options:\n"));
printf(_(" --redundancy specifies how many full backups purge command should keep\n"));
printf(_(" --window specifies the number of days of recoverability\n"));
}
static void

31
pg_probackup.h
View File

@ -34,8 +34,8 @@
#define BACKUPS_DIR "backups"
#define PG_XLOG_DIR "pg_xlog"
#define PG_TBLSPC_DIR "pg_tblspc"
#define BACKUP_INI_FILE "backup.conf"
#define PG_RMAN_INI_FILE "pg_probackup.conf"
#define BACKUP_CONF_FILE "backup.conf"
#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf"
#define MKDIRS_SH_FILE "mkdirs.sh"
#define DATABASE_FILE_LIST "file_database.txt"
#define PG_BACKUP_LABEL_FILE "backup_label"
@ -141,14 +141,6 @@ typedef struct pgBackup
time_t parent_backup;
} pgBackup;
typedef struct pgBackupOption
{
bool smooth_checkpoint;
int keep_data_generations;
int keep_data_days;
} pgBackupOption;
/* special values of pgBackup */
#define KEEP_INFINITE (INT_MAX)
#define BYTES_INVALID (-1)
@ -210,10 +202,15 @@ extern bool stream_wal;
extern bool from_replica;
extern bool progress;
extern bool delete_wal;
extern uint64 system_identifier;
/* retention configuration */
extern uint32 retention_redundancy;
extern uint32 retention_window;
/* in backup.c */
extern int do_backup(pgBackupOption bkupopt);
extern int do_backup(bool smooth_checkpoint);
extern BackupMode parse_backup_mode(const char *value);
extern void check_server_version(void);
extern bool fileExists(const char *path);
@ -243,11 +240,12 @@ extern int do_init(void);
/* in show.c */
extern int do_show(time_t backup_id);
extern int do_retention_show(void);
/* in delete.c */
extern int do_delete(time_t backup_id);
extern void pgBackupDelete(int keep_generations, int keep_days);
extern int do_deletewal(time_t backup_id, bool strict);
extern int do_retention_purge(void);
/* in fetch.c */
extern char *slurpFile(const char *datadir,
@ -266,17 +264,16 @@ extern void pgBackupValidate(pgBackup *backup,
bool size_only,
bool for_get_timeline);
/* in catalog.c */
extern pgBackup *catalog_get_backup(time_t timestamp);
extern pgBackup *read_backup(time_t timestamp);
extern void init_backup(pgBackup *backup);
extern parray *catalog_get_backup_list(time_t backup_id);
extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
TimeLineID tli);
extern int catalog_lock(void);
extern int catalog_lock(bool check_catalog);
extern void catalog_unlock(void);
extern void catalog_init_config(pgBackup *backup);
extern void pgBackupWriteConfigSection(FILE *out, pgBackup *backup);
extern void pgBackupWriteResultSection(FILE *out, pgBackup *backup);
extern void pgBackupWriteIni(pgBackup *backup);

26
pgut/pgut.c
View File

@ -71,15 +71,15 @@ static const char *get_username(void);
static pgut_option default_options[] =
{
{ 's', 'd', "dbname" , &pgut_dbname },
{ 's', 'h', "host" , &host },
{ 's', 'p', "port" , &port },
{ 'b', 'q', "quiet" , &quiet },
{ 's', 'U', "username" , &username },
{ 'b', 'v', "verbose" , &verbose },
{ 's', 'd', "dbname" , &pgut_dbname, SOURCE_CMDLINE },
{ 's', 'h', "host" , &host, SOURCE_CMDLINE },
{ 's', 'p', "port" , &port, SOURCE_CMDLINE },
{ 'b', 'q', "quiet" , &quiet, SOURCE_CMDLINE },
{ 's', 'U', "username" , &username, SOURCE_CMDLINE },
{ 'b', 'v', "verbose" , &verbose, SOURCE_CMDLINE },
#ifndef PGUT_NO_PROMPT
{ 'Y', 'w', "no-password" , &prompt_password },
{ 'y', 'W', "password" , &prompt_password },
{ 'Y', 'w', "no-password" , &prompt_password, SOURCE_CMDLINE },
{ 'y', 'W', "password" , &prompt_password, SOURCE_CMDLINE },
#endif
{ 0 }
};
@ -576,8 +576,7 @@ option_from_env(pgut_option options[])
const char *s;
const char *value;
if (opt->source > SOURCE_ENV ||
opt->allowed == SOURCE_DEFAULT || opt->allowed > SOURCE_ENV)
if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV)
continue;
for (s = opt->lname, j = 0; *s && j < lengthof(name) - 1; s++, j++)
@ -632,6 +631,9 @@ pgut_getopt(int argc, char **argv, pgut_option options[])
while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1)
{
opt = option_find(c, default_options, options);
if (opt->allowed < SOURCE_CMDLINE)
elog(ERROR, "option %s cannot be specified in command line",
opt->lname);
assign_option(opt, optarg, SOURCE_CMDLINE);
}
@ -698,8 +700,8 @@ pgut_readopt(const char *path, pgut_option options[], int elevel)
if (key_equals(key, opt->lname))
{
if (opt->allowed == SOURCE_DEFAULT ||
opt->allowed > SOURCE_FILE)
if (opt->allowed < SOURCE_FILE &&
opt->allowed != SOURCE_FILE_STRICT)
elog(elevel, "option %s cannot specified in file", opt->lname);
else if (opt->source <= SOURCE_FILE)
assign_option(opt, value, SOURCE_FILE);

1
pgut/pgut.h
View File

@ -40,6 +40,7 @@ typedef enum YesNo
typedef enum pgut_optsrc
{
SOURCE_DEFAULT,
SOURCE_FILE_STRICT,
SOURCE_ENV,
SOURCE_FILE,
SOURCE_CMDLINE,

81
restore.c
View File

@ -30,9 +30,6 @@ static void create_recovery_conf(time_t backup_id,
const char *target_inclusive,
TimeLineID target_tli);
static void print_backup_lsn(const pgBackup *backup);
static void search_next_wal(const char *path,
XLogRecPtr *need_lsn,
parray *timelines);
static void restore_files(void *arg);
@ -48,7 +45,6 @@ do_restore(time_t backup_id,
{
int i;
int base_index; /* index of base (full) backup */
int last_restored_index; /* index of last restored database backup */
int ret;
TimeLineID cur_tli;
TimeLineID backup_tli;
@ -60,7 +56,6 @@ do_restore(time_t backup_id,
pgBackup *base_backup = NULL;
pgBackup *dest_backup = NULL;
pgRecoveryTarget *rt = NULL;
XLogRecPtr need_lsn;
bool backup_id_found = false;
/* PGDATA and ARCLOG_PATH are always required */
@ -72,12 +67,12 @@ do_restore(time_t backup_id,
elog(LOG, "restore start");
/* get exclusive lock of backup catalog */
ret = catalog_lock();
ret = catalog_lock(false);
if (ret == -1)
elog(ERROR, "cannot lock backup catalog.");
else if (ret == 1)
elog(ERROR,
"another pg_probackup is running, stop restore.");
"another pg_probackup is running, stop restore.");
/* confirm the PostgreSQL server is not running */
if (is_pg_running())
@ -185,8 +180,6 @@ base_backup_found:
/* restore base backup */
restore_database(base_backup);
last_restored_index = base_index;
/* restore following differential backup */
elog(LOG, "searching differential backup...");
@ -220,19 +213,8 @@ base_backup_found:
print_backup_lsn(backup);
restore_database(backup);
last_restored_index = i;
}
if (!stream_wal || target_time != NULL || target_xid != NULL)
for (i = last_restored_index; i >= 0; i--)
{
elog(LOG, "searching archived WAL...");
search_next_wal(arclog_path, &need_lsn, timelines);
elog(LOG, "all necessary files are found");
}
/* create recovery.conf */
if (!stream_wal || target_time != NULL || target_xid != NULL)
create_recovery_conf(backup_id, target_time, target_xid, target_inclusive, target_tli);
@ -713,65 +695,6 @@ print_backup_lsn(const pgBackup *backup)
(uint32) backup->stop_lsn);
}
static void
search_next_wal(const char *path, XLogRecPtr *need_lsn, parray *timelines)
{
int i;
int j;
int count;
char xlogfname[MAXFNAMELEN];
char pre_xlogfname[MAXFNAMELEN];
char xlogpath[MAXPGPATH];
struct stat st;
count = 0;
for (;;)
{
for (i = 0; i < parray_num(timelines); i++)
{
pgTimeLine *timeline = (pgTimeLine *) parray_get(timelines, i);
XLogSegNo targetSegNo;
XLByteToSeg(*need_lsn, targetSegNo);
XLogFileName(xlogfname, timeline->tli, targetSegNo);
join_path_components(xlogpath, path, xlogfname);
if (stat(xlogpath, &st) == 0)
break;
}
/* not found */
if (i == parray_num(timelines))
{
if (count == 1)
elog(LOG, "\n");
else if (count > 1)
elog(LOG, " - %s", pre_xlogfname);
return;
}
count++;
if (count == 1)
elog(LOG, "%s", xlogfname);
strcpy(pre_xlogfname, xlogfname);
/* delete old TLI */
for (j = i + 1; j < parray_num(timelines); j++)
parray_remove(timelines, i + 1);
/* XXX: should we add a linebreak when we find a timeline? */
/*
* Move to next xlog segment. Note that we need to increment
* by XLogSegSize to jump directly to the next WAL segment file
* and as this value is used to generate the WAL file name with
* the current timeline of backup.
*/
*need_lsn += XLogSegSize;
}
}
pgRecoveryTarget *
checkIfCreateRecoveryConf(const char *target_time,
const char *target_xid,

17
show.c
View File

@ -29,7 +29,7 @@ do_show(time_t backup_id)
{
pgBackup *backup;
backup = catalog_get_backup(backup_id);
backup = read_backup(backup_id);
if (backup == NULL)
{
char timestamp[100];
@ -62,6 +62,21 @@ do_show(time_t backup_id)
return 0;
}
/*
* Show backup catalog retention policy configuration.
*/
int
do_retention_show(void)
{
fprintf(stdout, "# retention policy\n");
if (retention_redundancy > 0)
fprintf(stdout, "REDUNDANCY=%u\n", retention_redundancy);
if (retention_window > 0)
fprintf(stdout, "WINDOW=%u\n", retention_window);
return 0;
}
static void
pretty_size(int64 size, char *buf, size_t len)
{

2
status.c
View File

@ -2,7 +2,7 @@
*
* status.c
*
* Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
* Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
*
* Monitor status of a PostgreSQL server.
*

5
tests/__init__.py
View File

@ -1,6 +1,8 @@
import unittest
from . import init_test, option_test, show_test, backup_test, delete_test, restore_test, validate_test
from . import init_test, option_test, show_test, \
backup_test, delete_test, restore_test, validate_test, \
retention_test
def load_tests(loader, tests, pattern):
@ -12,5 +14,6 @@ def load_tests(loader, tests, pattern):
suite.addTests(loader.loadTestsFromModule(delete_test))
suite.addTests(loader.loadTestsFromModule(restore_test))
suite.addTests(loader.loadTestsFromModule(validate_test))
suite.addTests(loader.loadTestsFromModule(retention_test))
return suite

7
tests/expected/option_help.out
View File

@ -3,11 +3,12 @@ pg_probackup manage backup/recovery of PostgreSQL database.
Usage:
pg_probackup [option...] init
pg_probackup [option...] backup
pg_probackup [option...] restore
pg_probackup [option...] restore [backup-ID]
pg_probackup [option...] show [backup-ID]
pg_probackup [option...] validate backup-ID
pg_probackup [option...] delete backup-ID
pg_probackup [option...] delwal [backup-ID]
pg_probackup [option...] retention show|purge
Common Options:
-B, --backup-path=PATH location of the backup storage area
@ -33,6 +34,10 @@ Restore options:
Delete options:
--wal remove unnecessary wal files
Retention options:
--redundancy specifies how many full backups purge command should keep
--window specifies the number of days of recoverability
Connection options:
-d, --dbname=DBNAME database to connect
-h, --host=HOSTNAME database server host or socket directory

8
tests/pb_lib.py
View File

@ -187,6 +187,14 @@ class ProbackupTest(object):
# print(cmd_list)
return self.run_pb(options + cmd_list)
def retention_purge_pb(self, node, options=[]):
cmd_list = [
"-B", self.backup_dir(node),
"retention", "purge",
]
return self.run_pb(options + cmd_list)
def get_control_data(self, node):
pg_controldata = node.get_bin_path("pg_controldata")
out_data = {}

66
tests/retention_test.py Normal file
View File

@ -0,0 +1,66 @@
import unittest
import os
from datetime import datetime, timedelta
from os import path
from .pb_lib import ProbackupTest
from testgres import stop_all
class RetentionTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(RetentionTest, self).__init__(*args, **kwargs)
@classmethod
def tearDownClass(cls):
stop_all()
def test_retention_redundancy_1(self):
"""purge backups using redundancy-based retention policy"""
node = self.make_bnode('retention_redundancy_1',
base_dir="tmp_dirs/retention/retention_redundancy_1")
node.start()
self.init_pb(node)
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
conf.write("REDUNDANCY=1\n")
# Make backups to be purged
self.backup_pb(node)
self.backup_pb(node, backup_type="page")
# Make backups to be keeped
self.backup_pb(node)
self.backup_pb(node, backup_type="page")
self.assertEqual(len(self.show_pb(node)), 4)
# Purge backups
self.retention_purge_pb(node)
self.assertEqual(len(self.show_pb(node)), 2)
node.stop()
def test_retention_window_2(self):
"""purge backups using window-based retention policy"""
node = self.make_bnode('retention_window_2',
base_dir="tmp_dirs/retention/retention_window_2")
node.start()
self.init_pb(node)
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
conf.write("REDUNDANCY=1\n")
conf.write("WINDOW=2\n")
# All backups will be keeped
self.backup_pb(node)
self.backup_pb(node, backup_type="page")
self.backup_pb(node)
self.backup_pb(node, backup_type="page")
self.assertEqual(len(self.show_pb(node)), 4)
# Purge backups
self.retention_purge_pb(node)
self.assertEqual(len(self.show_pb(node)), 4)
node.stop()

2
tests/validate_test.py
View File

@ -58,4 +58,4 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
id_backup = self.show_pb(node)[0].id
res = self.validate_pb(node, id_backup, options=['--xid=%s' % target_xid])
self.assertIn(six.b("incorrect resource manager data checksum in record"), res)
self.assertIn(six.b("could not read WAL record at"), res)

4
validate.c
View File

@ -30,7 +30,7 @@ void do_validate_last(void)
parray *backup_list;
bool another_pg_probackup = false;
ret = catalog_lock();
ret = catalog_lock(false);
if (ret == 1)
another_pg_probackup = true;
@ -85,7 +85,7 @@ int do_validate(time_t backup_id,
pgBackup *base_backup = NULL;
bool backup_id_found = false;
catalog_lock();
catalog_lock(false);
rt = checkIfCreateRecoveryConf(target_time, target_xid, target_inclusive);
if (rt == NULL)