db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-07 13:40:17 +02:00
Code Issues Releases Activity

Merge branch 'master' into pgpro-2065

Browse Source
This commit is contained in:
Grigory Smolkin 2019-04-03 04:00:54 +03:00
commit db3273b344
10 changed files with 594 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/backup.c
View File

@ -1325,8 +1325,8 @@ do_backup(time_t start_time)
* After successfil backup completion remove backups
* which are expired according to retention policies
*/
if (delete_expired || delete_wal)
do_retention_purge();
if (delete_expired || merge_expired || delete_wal)
do_retention();
return 0;
}

62
src/catalog.c
View File

@ -1009,6 +1009,32 @@ pgBackupGetPath2(const pgBackup *backup, char *path, size_t len,
make_native_path(path);
}
/*
* Check if multiple backups consider target backup to be their direct parent
*/
bool
is_prolific(parray *backup_list, pgBackup *target_backup)
{
int i;
int child_counter = 0;
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i);
/* consider only OK and DONE backups */
if (tmp_backup->parent_backup == target_backup->start_time &&
(tmp_backup->status == BACKUP_STATUS_OK ||
tmp_backup->status == BACKUP_STATUS_DONE))
child_counter++;
}
if (child_counter > 1)
return true;
else
return false;
}
/*
* Find parent base FULL backup for current backup using parent_backup_link
*/
@ -1018,6 +1044,7 @@ find_parent_full_backup(pgBackup *current_backup)
pgBackup *base_full_backup = NULL;
base_full_backup = current_backup;
/* sanity */
if (!current_backup)
elog(ERROR, "Target backup cannot be NULL");
@ -1040,6 +1067,35 @@ find_parent_full_backup(pgBackup *current_backup)
return base_full_backup;
}
/*
* Find closest child of target_backup. If there are several direct
* offsprings in backup_list, then first win.
*/
pgBackup*
find_direct_child(parray *backup_list, pgBackup *target_backup)
{
int i;
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i);
if (tmp_backup->backup_mode == BACKUP_MODE_FULL)
continue;
/* Consider only OK and DONE children */
if (tmp_backup->parent_backup == target_backup->start_time &&
(tmp_backup->status == BACKUP_STATUS_OK ||
tmp_backup->status == BACKUP_STATUS_DONE))
{
return tmp_backup;
}
}
elog(WARNING, "Failed to find a direct child for backup %s",
base36enc(target_backup->start_time));
return NULL;
}
/*
* Interate over parent chain and look for any problems.
* Return 0 if chain is broken.
@ -1146,6 +1202,6 @@ get_backup_index_number(parray *backup_list, pgBackup *backup)
if (tmp_backup->start_time == backup->start_time)
return i;
}
elog(ERROR, "Failed to find backup %s", base36enc(backup->start_time));
return 0;
}
elog(WARNING, "Failed to find backup %s", base36enc(backup->start_time));
return -1;
}

591
src/delete.c
View File

@ -16,6 +16,15 @@
static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
uint32 xlog_seg_size);
static void do_retention_internal(parray *backup_list, parray *to_keep_list,
parray *to_purge_list);
static void do_retention_merge(parray *backup_list, parray *to_keep_list,
parray *to_purge_list);
static void do_retention_purge(parray *to_keep_list, parray *to_purge_list);
static void do_retention_wal(void);
static bool backup_deleted = false; /* At least one backup was deleted */
static bool backup_merged = false; /* At least one merge was enacted */
void
do_delete(time_t backup_id)
@ -102,141 +111,511 @@ do_delete(time_t backup_id)
}
/*
* Remove backups by retention policy. Retention policy is configured by
* Merge and purge backups by retention policy. Retention policy is configured by
* retention_redundancy and retention_window variables.
*
* Invalid backups handled in Oracle style, so invalid backups are ignored
* for the purpose of retention fulfillment,
* i.e. CORRUPT full backup do not taken in account when deteremine
* which FULL backup should be keeped for redundancy obligation(only valid do),
* but if invalid backup is not guarded by retention - it is removed
*/
int
do_retention_purge(void)
int do_retention(void)
{
parray *backup_list;
size_t i;
XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
TimeLineID oldest_tli = 0;
bool backup_deleted = false; /* At least one backup was deleted */
parray *backup_list = NULL;
parray *to_keep_list = parray_new();
parray *to_purge_list = parray_new();
if (delete_expired)
bool retention_is_set = false; /* At least one retention policy is set */
bool backup_list_is_empty = false;
/* Get a complete list of backups. */
backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
if (parray_num(backup_list) == 0)
backup_list_is_empty = true;
if (delete_expired || merge_expired)
{
if (instance_config.retention_redundancy > 0)
elog(LOG, "REDUNDANCY=%u", instance_config.retention_redundancy);
if (instance_config.retention_window > 0)
elog(LOG, "WINDOW=%u", instance_config.retention_window);
if (instance_config.retention_redundancy == 0
&& instance_config.retention_window == 0)
if (instance_config.retention_redundancy == 0 &&
instance_config.retention_window == 0)
{
/* Retention is disabled but we still can cleanup wal */
elog(WARNING, "Retention policy is not set");
if (!delete_wal)
return 0;
}
else
/* At least one retention policy is active */
retention_is_set = true;
}
/* Get a complete list of backups. */
backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
if (parray_num(backup_list) == 0)
{
elog(INFO, "backup list is empty, purging won't be executed");
return 0;
}
if (retention_is_set && backup_list_is_empty)
elog(WARNING, "Backup list is empty, retention purge and merge are problematic");
/* Find target backups to be deleted */
if (delete_expired &&
(instance_config.retention_redundancy > 0 ||
instance_config.retention_window > 0))
{
bool keep_next_backup = false; /* Do not delete first full backup */
time_t days_threshold;
uint32 backup_num = 0;
/* Populate purge and keep lists, and show retention state messages */
if (retention_is_set && !backup_list_is_empty)
do_retention_internal(backup_list, to_keep_list, to_purge_list);
days_threshold = time(NULL) -
(instance_config.retention_window * 60 * 60 * 24);
if (merge_expired && !dry_run && !backup_list_is_empty)
do_retention_merge(backup_list, to_keep_list, to_purge_list);
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
uint32 backup_num_evaluate = backup_num;
if (delete_expired && !dry_run && !backup_list_is_empty)
do_retention_purge(to_keep_list, to_purge_list);
/* Consider only validated and correct backups */
if (backup->status != BACKUP_STATUS_OK)
continue;
/*
* When a valid full backup was found, we can delete the
* backup that is older than it using the number of generations.
*/
if (backup->backup_mode == BACKUP_MODE_FULL)
backup_num++;
/* TODO: some sort of dry run for delete_wal */
if (delete_wal && !dry_run)
do_retention_wal();
/* Evaluate retention_redundancy if this backup is eligible for removal */
if (keep_next_backup ||
instance_config.retention_redundancy >= backup_num_evaluate + 1 ||
(instance_config.retention_window > 0 &&
backup->recovery_time >= days_threshold))
{
/* Save LSN and Timeline to remove unnecessary WAL segments */
oldest_lsn = backup->start_lsn;
oldest_tli = backup->tli;
/* Save parent backup of this incremental backup */
if (backup->backup_mode != BACKUP_MODE_FULL)
keep_next_backup = true;
/*
* Previous incremental backup was kept or this is first backup
* so do not delete this backup.
*/
else
keep_next_backup = false;
continue;
}
/*
* If the backup still is used do not interrupt go to the next
* backup.
*/
if (!lock_backup(backup))
{
elog(WARNING, "Cannot lock backup %s directory, skip purging",
base36enc(backup->start_time));
continue;
}
/* Delete backup and update status to DELETED */
delete_backup_files(backup);
backup_deleted = true;
keep_next_backup = false; /* reset it */
}
}
/*
* If oldest_lsn and oldest_tli weren`t set because previous step was skipped
* then set them now if we are going to purge WAL
*/
if (delete_wal && (XLogRecPtrIsInvalid(oldest_lsn)))
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) - 1);
oldest_lsn = backup->start_lsn;
oldest_tli = backup->tli;
}
/* Be paranoid */
if (XLogRecPtrIsInvalid(oldest_lsn))
elog(ERROR, "Not going to purge WAL because LSN is invalid");
/* Purge WAL files */
if (delete_wal)
{
delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size);
}
/* Cleanup */
parray_walk(backup_list, pgBackupFree);
parray_free(backup_list);
if (!backup_merged)
elog(INFO, "There are no backups to merge by retention policy");
if (backup_deleted)
elog(INFO, "Purging finished");
else
elog(INFO, "There are no backups to delete by retention policy");
/* Cleanup */
parray_walk(backup_list, pgBackupFree);
parray_free(backup_list);
parray_free(to_keep_list);
parray_free(to_purge_list);
return 0;
}
/* Evaluate every backup by retention policies and populate purge and keep lists.
* Also for every backup print its status ('Active' or 'Expired') according
* to active retention policies.
*/
static void
do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list)
{
int i;
time_t current_time;
/* For retention calculation */
uint32 n_full_backups = 0;
int cur_full_backup_num = 0;
time_t days_threshold = 0;
/* For fancy reporting */
float actual_window = 0;
/* Get current time */
current_time = time(NULL);
/* Calculate n_full_backups and days_threshold */
if (instance_config.retention_redundancy > 0)
{
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
/* Consider only valid backups for Redundancy */
if (instance_config.retention_redundancy > 0 &&
backup->backup_mode == BACKUP_MODE_FULL &&
(backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE))
{
n_full_backups++;
}
}
}
if (instance_config.retention_window > 0)
{
days_threshold = current_time -
(instance_config.retention_window * 60 * 60 * 24);
}
elog(INFO, "Evaluate backups by retention");
for (i = (int) parray_num(backup_list) - 1; i >= 0; i--)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i);
/* Remember the serial number of latest valid FULL backup */
if (backup->backup_mode == BACKUP_MODE_FULL &&
(backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE))
{
cur_full_backup_num++;
}
/* Check if backup in needed by retention policy */
if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) &&
(instance_config.retention_redundancy <= (n_full_backups - cur_full_backup_num)))
{
/* This backup is not guarded by retention
*
* Redundancy = 1
* FULL CORRUPT in retention (not count toward redundancy limit)
* FULL in retention
* ------retention redundancy -------
* PAGE3 in retention
* ------retention window -----------
* PAGE2 out of retention
* PAGE1 out of retention
* FULL out of retention <- We are here
* FULL CORRUPT out of retention
*/
/* Add backup to purge_list */
elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time));
parray_append(to_purge_list, backup);
continue;
}
}
/* sort keep_list and purge list */
parray_qsort(to_keep_list, pgBackupCompareIdDesc);
parray_qsort(to_purge_list, pgBackupCompareIdDesc);
/* FULL
* PAGE
* PAGE <- Only such backups must go into keep list
---------retention window ----
* PAGE
* FULL
* PAGE
* FULL
*/
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
/* Do not keep invalid backups by retention */
if (backup->status != BACKUP_STATUS_OK &&
backup->status != BACKUP_STATUS_DONE)
continue;
/* only incremental backups should be in keep list */
if (backup->backup_mode == BACKUP_MODE_FULL)
continue;
/* orphan backup cannot be in keep list */
if (!backup->parent_backup_link)
continue;
/* skip if backup already in purge list */
if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc))
continue;
/* if parent in purge_list, add backup to keep list */
if (parray_bsearch(to_purge_list,
backup->parent_backup_link,
pgBackupCompareIdDesc))
{
/* make keep list a bit sparse */
parray_append(to_keep_list, backup);
continue;
}
}
/* Message about retention state of backups
* TODO: Float is ugly, rewrite somehow.
*/
cur_full_backup_num = 1;
for (i = 0; i < parray_num(backup_list); i++)
{
char *action = "Active";
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc))
action = "Expired";
if (backup->recovery_time == 0)
actual_window = 0;
else
actual_window = ((float)current_time - (float)backup->recovery_time)/(60 * 60 * 24);
elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %.2fd/%ud. %s",
base36enc(backup->start_time),
pgBackupGetBackupMode(backup),
status2str(backup->status),
cur_full_backup_num,
instance_config.retention_redundancy,
actual_window, instance_config.retention_window,
action);
if (backup->backup_mode == BACKUP_MODE_FULL)
cur_full_backup_num++;
}
}
/* Merge partially expired incremental chains */
static void
do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list)
{
int i;
int j;
/* IMPORTANT: we can merge to only those FULL backup, that is NOT
* guarded by retention and final target of such merge must be
* an incremental backup that is guarded by retention !!!
*
* PAGE4 E
* PAGE3 D
--------retention window ---
* PAGE2 C
* PAGE1 B
* FULL A
*
* after retention merge:
* PAGE4 E
* FULL D
*/
/* Merging happens here */
for (i = 0; i < parray_num(to_keep_list); i++)
{
char *keep_backup_id = NULL;
pgBackup *full_backup = NULL;
parray *merge_list = NULL;
pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i);
/* keep list may shrink during merge */
if (!keep_backup)
continue;
elog(INFO, "Consider backup %s for merge", base36enc(keep_backup->start_time));
/* Got valid incremental backup, find its FULL ancestor */
full_backup = find_parent_full_backup(keep_backup);
/* Failed to find parent */
if (!full_backup)
{
elog(WARNING, "Failed to find FULL parent for %s", base36enc(keep_backup->start_time));
continue;
}
/* Check that ancestor is in purge_list */
if (!parray_bsearch(to_purge_list,
full_backup,
pgBackupCompareIdDesc))
{
elog(WARNING, "Skip backup %s for merging, "
"because his FULL parent is not marked for purge", base36enc(keep_backup->start_time));
continue;
}
/* FULL backup in purge list, thanks to sparsing of keep_list current backup is
* final target for merge, but there could be intermediate incremental
* backups from purge_list.
*/
keep_backup_id = base36enc_dup(keep_backup->start_time);
elog(INFO, "Merge incremental chain between FULL backup %s and backup %s",
base36enc(full_backup->start_time), keep_backup_id);
pg_free(keep_backup_id);
merge_list = parray_new();
/* Form up a merge list */
while(keep_backup->parent_backup_link)
{
parray_append(merge_list, keep_backup);
keep_backup = keep_backup->parent_backup_link;
}
/* sanity */
if (!merge_list)
continue;
/* sanity */
if (parray_num(merge_list) == 0)
{
parray_free(merge_list);
continue;
}
/* In the end add FULL backup for easy locking */
parray_append(merge_list, full_backup);
/* Remove FULL backup from purge list */
parray_rm(to_purge_list, full_backup, pgBackupCompareId);
/* Lock merge chain */
catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0);
/* Merge list example:
* 0 PAGE3
* 1 PAGE2
* 2 PAGE1
* 3 FULL
*
* Сonsequentially merge incremental backups from PAGE1 to PAGE3
* into FULL.
*/
for (j = parray_num(merge_list) - 2; j >= 0; j--)
{
pgBackup *from_backup = (pgBackup *) parray_get(merge_list, j);
/* Consider this extreme case */
// PAGEa1 PAGEb1 both valid
// \ /
// FULL
/* Check that FULL backup do not has multiple descendants
* full_backup always point to current full_backup after merge
*/
if (is_prolific(backup_list, full_backup))
{
elog(WARNING, "Backup %s has multiple valid descendants. "
"Automatic merge is not possible.", base36enc(full_backup->start_time));
break;
}
merge_backups(full_backup, from_backup);
backup_merged = true;
/* Try to remove merged incremental backup from both keep and purge lists */
parray_rm(to_purge_list, from_backup, pgBackupCompareId);
parray_set(to_keep_list, i, NULL);
}
/* Cleanup */
parray_free(merge_list);
}
elog(INFO, "Retention merging finished");
}
/* Purge expired backups */
static void
do_retention_purge(parray *to_keep_list, parray *to_purge_list)
{
int i;
int j;
/* Remove backups by retention policy. Retention policy is configured by
* retention_redundancy and retention_window
* Remove only backups, that do not have children guarded by retention
*
* TODO: We do not consider the situation if child is marked for purge
* but parent isn`t. Maybe something bad happened with time on server?
*/
for (j = 0; j < parray_num(to_purge_list); j++)
{
bool purge = true;
pgBackup *delete_backup = (pgBackup *) parray_get(to_purge_list, j);
elog(LOG, "Consider backup %s for purge",
base36enc(delete_backup->start_time));
/* Evaluate marked for delete backup against every backup in keep list.
* If marked for delete backup is recognized as parent of one of those,
* then this backup should not be deleted.
*/
for (i = 0; i < parray_num(to_keep_list); i++)
{
char *keeped_backup_id;
pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i);
/* item could have been nullified in merge */
if (!keep_backup)
continue;
/* Full backup cannot be a descendant */
if (keep_backup->backup_mode == BACKUP_MODE_FULL)
continue;
keeped_backup_id = base36enc_dup(keep_backup->start_time);
elog(LOG, "Check if backup %s is parent of backup %s",
base36enc(delete_backup->start_time), keeped_backup_id);
if (is_parent(delete_backup->start_time, keep_backup, true))
{
/* We must not delete this backup, evict it from purge list */
elog(LOG, "Retain backup %s from purge because his "
"descendant %s is guarded by retention",
base36enc(delete_backup->start_time), keeped_backup_id);
purge = false;
pg_free(keeped_backup_id);
break;
}
pg_free(keeped_backup_id);
}
/* Retain backup */
if (!purge)
continue;
/* Actual purge */
if (!lock_backup(delete_backup))
{
/* If the backup still is used, do not interrupt and go to the next */
elog(WARNING, "Cannot lock backup %s directory, skip purging",
base36enc(delete_backup->start_time));
continue;
}
/* Delete backup and update status to DELETED */
delete_backup_files(delete_backup);
backup_deleted = true;
}
}
/* Purge WAL */
static void
do_retention_wal(void)
{
parray *backup_list = NULL;
XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
TimeLineID oldest_tli = 0;
bool backup_list_is_empty = false;
/* Get list of backups. */
backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
if (parray_num(backup_list) == 0)
backup_list_is_empty = true;
/* Save LSN and Timeline to remove unnecessary WAL segments */
if (!backup_list_is_empty)
{
pgBackup *backup = NULL;
/* Get LSN and TLI of oldest alive backup */
backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) -1);
oldest_tli = backup->tli;
oldest_lsn = backup->start_lsn;
}
/* Be paranoid */
if (!backup_list_is_empty && XLogRecPtrIsInvalid(oldest_lsn))
elog(ERROR, "Not going to purge WAL because LSN is invalid");
/* Purge WAL files */
delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size);
/* Cleanup */
parray_walk(backup_list, pgBackupFree);
parray_free(backup_list);
}
/*

17
src/help.c
View File

@ -110,7 +110,7 @@ help_pg_probackup(void)
printf(_(" [--log-directory=log-directory]\n"));
printf(_(" [--log-rotation-size=log-rotation-size]\n"));
printf(_(" [--log-rotation-age=log-rotation-age]\n"));
printf(_(" [--delete-expired] [--delete-wal]\n"));
printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n"));
printf(_(" [--retention-redundancy=retention-redundancy]\n"));
printf(_(" [--retention-window=retention-window]\n"));
printf(_(" [--compress]\n"));
@ -153,7 +153,8 @@ help_pg_probackup(void)
printf(_(" [--format=format]\n"));
printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" [--wal] [-i backup-id | --expired]\n"));
printf(_(" [--wal] [-i backup-id | --expired | --merge-expired]\n"));
printf(_(" [--dry-run]\n"));
printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" -i backup-id [--progress] [-j num-threads]\n"));
@ -208,7 +209,7 @@ help_backup(void)
printf(_(" [--log-directory=log-directory]\n"));
printf(_(" [--log-rotation-size=log-rotation-size]\n"));
printf(_(" [--log-rotation-age=log-rotation-age]\n"));
printf(_(" [--delete-expired] [--delete-wal]\n"));
printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n"));
printf(_(" [--retention-redundancy=retention-redundancy]\n"));
printf(_(" [--retention-window=retention-window]\n"));
printf(_(" [--compress]\n"));
@ -262,11 +263,14 @@ help_backup(void)
printf(_("\n Retention options:\n"));
printf(_(" --delete-expired delete backups expired according to current\n"));
printf(_(" retention policy after successful backup completion\n"));
printf(_(" --merge-expired merge backups expired according to current\n"));
printf(_(" retention policy after successful backup completion\n"));
printf(_(" --delete-wal remove redundant archived wal files\n"));
printf(_(" --retention-redundancy=retention-redundancy\n"));
printf(_(" number of full backups to keep; 0 disables; (default: 0)\n"));
printf(_(" --retention-window=retention-window\n"));
printf(_(" number of days of recoverability; 0 disables; (default: 0)\n"));
printf(_(" --dry-run perform a trial run without any changes\n"));
printf(_("\n Compression options:\n"));
printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n"));
@ -474,14 +478,19 @@ static void
help_delete(void)
{
printf(_("%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" [-i backup-id | --expired] [--wal]\n\n"));
printf(_(" [-i backup-id | --expired | --merge-expired] [--wal]\n"));
printf(_(" [-j num-threads] [--dry-run]\n\n"));
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
printf(_(" --instance=instance_name name of the instance\n"));
printf(_(" -i, --backup-id=backup-id backup to delete\n"));
printf(_(" --expired delete backups expired according to current\n"));
printf(_(" retention policy\n"));
printf(_(" --merge-expired merge backups expired according to current\n"));
printf(_(" retention policy\n"));
printf(_(" --wal remove unnecessary wal files in WAL ARCHIVE\n"));
printf(_(" -j, --threads=NUM number of parallel threads\n"));
printf(_(" --dry-run perform a trial run without any changes\n"));
printf(_("\n Logging options:\n"));
printf(_(" --log-level-console=log-level-console\n"));

5
src/merge.c
View File

@ -34,7 +34,6 @@ typedef struct
int ret;
} merge_files_arg;
static void merge_backups(pgBackup *backup, pgBackup *next_backup);
static void *merge_files(void *arg);
static void
reorder_external_dirs(pgBackup *to_backup, parray *to_external,
@ -164,7 +163,7 @@ do_merge(time_t backup_id)
* - remove unnecessary directories and files from to_backup
* - update metadata of from_backup, it becames FULL backup
*/
static void
void
merge_backups(pgBackup *to_backup, pgBackup *from_backup)
{
char *to_backup_id = base36enc_dup(to_backup->start_time),
@ -657,7 +656,7 @@ merge_files(void *arg)
file->compress_alg = to_backup->compress_alg;
if (file->write_size != BYTES_INVALID)
elog(LOG, "Merged file \"%s\": " INT64_FORMAT " bytes",
elog(VERBOSE, "Merged file \"%s\": " INT64_FORMAT " bytes",
file->path, file->write_size);
/* Restore relative path */

17
src/pg_probackup.c
View File

@ -98,7 +98,9 @@ bool amcheck_parent = false;
/* delete options */
bool delete_wal = false;
bool delete_expired = false;
bool merge_expired = false;
bool force_delete = false;
bool dry_run = false;
/* compression options */
bool compress_shortcut = false;
@ -146,6 +148,8 @@ static ConfigOption cmd_options[] =
{ 'b', 234, "temp-slot", &temp_slot, SOURCE_CMD_STRICT },
{ 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT },
{ 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT },
{ 'b', 235, "merge-expired", &merge_expired, SOURCE_CMD_STRICT },
{ 'b', 237, "dry-run", &dry_run, SOURCE_CMD_STRICT },
/* TODO not completed feature. Make it unavailiable from user level
{ 'b', 18, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, */
/* restore options */
@ -556,12 +560,13 @@ main(int argc, char *argv[])
case DELETE_CMD:
if (delete_expired && backup_id_string)
elog(ERROR, "You cannot specify --delete-expired and --backup-id options together");
if (!delete_expired && !delete_wal && !backup_id_string)
elog(ERROR, "You must specify at least one of the delete options: --expired |--wal |--backup_id");
if (delete_wal && !delete_expired && !backup_id_string)
return do_retention_purge();
if (delete_expired)
return do_retention_purge();
if (merge_expired && backup_id_string)
elog(ERROR, "You cannot specify --merge-expired and --backup-id options together");
if (!delete_expired && !merge_expired && !delete_wal && !backup_id_string)
elog(ERROR, "You must specify at least one of the delete options: "
"--expired |--wal |--merge-expired |--delete-invalid |--backup_id");
if (!backup_id_string)
return do_retention();
else
do_delete(current.backup_id);
break;

9
src/pg_probackup.h
View File

@ -397,7 +397,9 @@ extern bool skip_external_dirs;
/* delete options */
extern bool delete_wal;
extern bool delete_expired;
extern bool merge_expired;
extern bool force_delete;
extern bool dry_run;
/* compression options */
extern bool compress_shortcut;
@ -445,6 +447,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions(
/* in merge.c */
extern void do_merge(time_t backup_id);
extern void merge_backups(pgBackup *backup, pgBackup *next_backup);
/* in init.c */
extern int do_init(void);
@ -467,7 +470,7 @@ extern int do_show(time_t requested_backup_id);
/* in delete.c */
extern void do_delete(time_t backup_id);
extern void delete_backup_files(pgBackup *backup);
extern int do_retention_purge(void);
extern int do_retention(void);
extern int do_delete_instance(void);
/* in fetch.c */
@ -512,10 +515,14 @@ extern void pgBackupInit(pgBackup *backup);
extern void pgBackupFree(void *backup);
extern int pgBackupCompareId(const void *f1, const void *f2);
extern int pgBackupCompareIdDesc(const void *f1, const void *f2);
extern int pgBackupCompareIdEqual(const void *l, const void *r);
extern pgBackup* find_direct_child(parray *backup_list, pgBackup *target_backup);
extern pgBackup* find_parent_full_backup(pgBackup *current_backup);
extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup);
extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive);
extern bool is_prolific(parray *backup_list, pgBackup *target_backup);
extern bool in_backup_list(parray *backup_list, pgBackup *target_backup);
extern int get_backup_index_number(parray *backup_list, pgBackup *backup);
#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS

2
src/restore.c
View File

@ -219,6 +219,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
}
}
}
pg_free(missing_backup_id);
/* No point in doing futher */
elog(ERROR, "%s of backup %s failed.", action, base36enc(dest_backup->start_time));
}
@ -255,6 +256,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
}
}
}
pg_free(parent_backup_id);
tmp_backup = find_parent_full_backup(dest_backup);
/* sanity */

14
src/validate.c
View File

@ -382,7 +382,6 @@ do_validate_all(void)
static void
do_validate_instance(void)
{
char *current_backup_id;
int i;
int j;
parray *backups;
@ -397,7 +396,6 @@ do_validate_instance(void)
for (i = 0; i < parray_num(backups); i++)
{
pgBackup *base_full_backup;
char *parent_backup_id;
current_backup = (pgBackup *) parray_get(backups, i);
@ -412,6 +410,7 @@ do_validate_instance(void)
/* chain is broken */
if (result == 0)
{
char *parent_backup_id;
/* determine missing backup ID */
parent_backup_id = base36enc_dup(tmp_backup->parent_backup);
@ -430,31 +429,31 @@ do_validate_instance(void)
elog(WARNING, "Backup %s has missing parent %s",
base36enc(current_backup->start_time), parent_backup_id);
}
pg_free(parent_backup_id);
continue;
}
/* chain is whole, but at least one parent is invalid */
else if (result == 1)
{
/* determine corrupt backup ID */
parent_backup_id = base36enc_dup(tmp_backup->start_time);
/* Oldest corrupt backup has a chance for revalidation */
if (current_backup->start_time != tmp_backup->start_time)
{
char *backup_id = base36enc_dup(tmp_backup->start_time);
/* orphanize current_backup */
if (current_backup->status == BACKUP_STATUS_OK)
{
write_backup_status(current_backup, BACKUP_STATUS_ORPHAN);
elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s",
base36enc(current_backup->start_time), parent_backup_id,
base36enc(current_backup->start_time), backup_id,
status2str(tmp_backup->status));
}
else
{
elog(WARNING, "Backup %s has parent %s with status: %s",
base36enc(current_backup->start_time),parent_backup_id,
base36enc(current_backup->start_time), backup_id,
status2str(tmp_backup->status));
}
pg_free(backup_id);
continue;
}
base_full_backup = find_parent_full_backup(current_backup);
@ -495,6 +494,7 @@ do_validate_instance(void)
*/
if (current_backup->status != BACKUP_STATUS_OK)
{
char *current_backup_id;
/* This is ridiculous but legal.
* PAGE_b2 <- OK
* PAGE_a2 <- OK

10
tests/retention.py
View File

@ -218,13 +218,13 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# count wal files in ARCHIVE
wals_dir = os.path.join(backup_dir, 'wal', 'node')
n_wals = len(os.listdir(wals_dir))
# n_wals = len(os.listdir(wals_dir))
self.assertTrue(n_wals > 0)
# self.assertTrue(n_wals > 0)
self.delete_expired(
backup_dir, 'node',
options=['--retention-window=1', '--expired', '--wal'])
# self.delete_expired(
# backup_dir, 'node',
# options=['--retention-window=1', '--expired', '--wal'])
# count again
n_wals = len(os.listdir(wals_dir))