Add retention show|purge commands. Add tests and documentation.

.gitignore

@@ -25,6 +25,9 @@
 /regression.diffs
 /regression.out
 /results
+/env
+/tests/__pycache__/
+/tests/tmp_dirs/
 
 # Extra files
 /datapagemap.c

backup.c

@@ -58,7 +58,7 @@ typedef struct
  */
 static void backup_cleanup(bool fatal, void *userdata);
 static void backup_files(void *arg);
-static parray *do_backup_database(parray *backup_list, pgBackupOption bkupopt);
+static parray *do_backup_database(parray *backup_list, bool smooth_checkpoint);
 static void confirm_block_size(const char *name, int blcksz);
 static void pg_start_backup(const char *label, bool smooth, pgBackup *backup);
 static void pg_stop_backup(pgBackup *backup);
@@ -96,7 +96,7 @@ static void StreamLog(void *arg);
  * Take a backup of database and return the list of files backed up.
  */
 static parray *
-do_backup_database(parray *backup_list, pgBackupOption bkupopt)
+do_backup_database(parray *backup_list, bool smooth_checkpoint)
 {
 	int			i;
 	parray	   *prev_files = NULL;	/* file list of previous database backup */
@@ -113,9 +113,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
 	backup_files_args *backup_threads_args[num_threads];
 	bool		is_ptrack_support;
 
-
 	/* repack the options */
-	bool	smooth_checkpoint = bkupopt.smooth_checkpoint;
 	pgBackup   *prev_backup = NULL;
 
 	/* Block backup operations on a standby */
@@ -446,15 +444,11 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
 
 
 int
-do_backup(pgBackupOption bkupopt)
+do_backup(bool smooth_checkpoint)
 {
-	parray *backup_list;
+	int			ret;
-	parray *files_database;
+	parray	   *backup_list;
-	int		ret;
+	parray	   *files_database;
-
-	/* repack the necessary options */
-	int keep_data_generations = bkupopt.keep_data_generations;
-	int keep_data_days        = bkupopt.keep_data_days;
 
 	/* PGDATA and BACKUP_MODE are always required */
 	if (pgdata == NULL)
@@ -481,12 +475,12 @@ do_backup(pgBackupOption bkupopt)
 	elog(LOG, "----------------------------------------");
 
 	/* get exclusive lock of backup catalog */
-	ret = catalog_lock();
+	ret = catalog_lock(true);
 	if (ret == -1)
 		elog(ERROR, "cannot lock backup catalog");
 	else if (ret == 1)
 		elog(ERROR,
-			"another pg_probackup is running, skipping this backup");
+			 "another pg_probackup is running, skipping this backup");
 
 	/* initialize backup result */
 	current.status = BACKUP_STATUS_RUNNING;
@@ -521,7 +515,7 @@ do_backup(pgBackupOption bkupopt)
 	pgut_atexit_push(backup_cleanup, NULL);
 
 	/* backup data */
-	files_database = do_backup_database(backup_list, bkupopt);
+	files_database = do_backup_database(backup_list, smooth_checkpoint);
 	pgut_atexit_pop(backup_cleanup, NULL);
 
 	/* update backup status to DONE */
@@ -550,10 +544,6 @@ do_backup(pgBackupOption bkupopt)
 		elog(LOG, "========================================");
 	}
 
-
-	/* Delete old backup files after all backup operation. */
-	pgBackupDelete(keep_data_generations, keep_data_days);
-
 	/* Cleanup backup mode file list */
 	if (files_database)
 		parray_walk(files_database, pgFileFree);

catalog.c

@@ -21,7 +21,7 @@
 
 #include "pgut/pgut-port.h"
 
-static pgBackup *catalog_read_ini(const char *path);
+static pgBackup *read_backup_from_file(const char *path);
 
 #define BOOL_TO_STR(val)	((val) ? "true" : "false")
 
@@ -32,17 +32,16 @@ static int lock_fd = -1;
  * If the lock is held by another one, return 1 immediately.
  */
 int
-catalog_lock(void)
+catalog_lock(bool check_catalog)
 {
-	int		ret;
+	int			ret;
-	char	id_path[MAXPGPATH];
+	char		id_path[MAXPGPATH];
 
-	join_path_components(id_path, backup_path, PG_RMAN_INI_FILE);
+	join_path_components(id_path, backup_path, BACKUP_CATALOG_CONF_FILE);
 	lock_fd = open(id_path, O_RDWR);
 	if (lock_fd == -1)
 		elog(errno == ENOENT ? ERROR : ERROR,
 			"cannot open file \"%s\": %s", id_path, strerror(errno));
-
 #ifdef __IBMC__
 	ret = lockf(lock_fd, LOCK_EX | LOCK_NB, 0);	/* non-blocking */
 #else
@@ -64,6 +63,19 @@ catalog_lock(void)
 		}
 	}
 
+	if (check_catalog)
+	{
+		uint64		id;
+
+		Assert(pgdata);
+
+		/* Check system-identifier */
+		id = get_system_identifier(true);
+		if (id != system_identifier)
+			elog(ERROR, "Backup directory was initialized for system id = %ld, but target system id = %ld",
+				 system_identifier, id);
+	}
+
 	return 0;
 }
 
@@ -82,15 +94,15 @@ catalog_unlock(void)
  * If no backup matches, return NULL.
  */
 pgBackup *
-catalog_get_backup(time_t timestamp)
+read_backup(time_t timestamp)
 {
 	pgBackup	tmp;
-	char		ini_path[MAXPGPATH];
+	char		conf_path[MAXPGPATH];
 
 	tmp.start_time = timestamp;
-	pgBackupGetPath(&tmp, ini_path, lengthof(ini_path), BACKUP_INI_FILE);
+	pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONF_FILE);
 
-	return catalog_read_ini(ini_path);
+	return read_backup_from_file(conf_path);
 }
 
 static bool
@@ -144,8 +156,8 @@ catalog_get_backup_list(time_t backup_id)
 		join_path_components(date_path, backups_path, date_ent->d_name);
 
 		/* read backup information from backup.ini */
-		snprintf(ini_path, MAXPGPATH, "%s/%s", date_path, BACKUP_INI_FILE);
+		snprintf(ini_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONF_FILE);
-		backup = catalog_read_ini(ini_path);
+		backup = read_backup_from_file(ini_path);
 
 		/* ignore corrupted backup */
 		if (backup)
@@ -309,7 +321,7 @@ pgBackupWriteIni(pgBackup *backup)
 	FILE   *fp = NULL;
 	char	ini_path[MAXPGPATH];
 
-	pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_INI_FILE);
+	pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_CONF_FILE);
 	fp = fopen(ini_path, "wt");
 	if (fp == NULL)
 		elog(ERROR, "cannot open INI file \"%s\": %s", ini_path,
@@ -330,7 +342,7 @@ pgBackupWriteIni(pgBackup *backup)
  *  - Do not care section.
  */
 static pgBackup *
-catalog_read_ini(const char *path)
+read_backup_from_file(const char *path)
 {
 	pgBackup   *backup;
 	char	   *backup_mode = NULL;
@@ -342,21 +354,21 @@ catalog_read_ini(const char *path)
 
 	pgut_option options[] =
 	{
-		{'s', 0, "backup-mode",			NULL, SOURCE_ENV},
+		{'s', 0, "backup-mode",			NULL, SOURCE_FILE_STRICT},
-		{'u', 0, "timelineid",			NULL, SOURCE_ENV},
+		{'u', 0, "timelineid",			NULL, SOURCE_FILE_STRICT},
-		{'s', 0, "start-lsn",			NULL, SOURCE_ENV},
+		{'s', 0, "start-lsn",			NULL, SOURCE_FILE_STRICT},
-		{'s', 0, "stop-lsn",			NULL, SOURCE_ENV},
+		{'s', 0, "stop-lsn",			NULL, SOURCE_FILE_STRICT},
-		{'t', 0, "start-time",			NULL, SOURCE_ENV},
+		{'t', 0, "start-time",			NULL, SOURCE_FILE_STRICT},
-		{'t', 0, "end-time",			NULL, SOURCE_ENV},
+		{'t', 0, "end-time",			NULL, SOURCE_FILE_STRICT},
-		{'U', 0, "recovery-xid",		NULL, SOURCE_ENV},
+		{'U', 0, "recovery-xid",		NULL, SOURCE_FILE_STRICT},
-		{'t', 0, "recovery-time",		NULL, SOURCE_ENV},
+		{'t', 0, "recovery-time",		NULL, SOURCE_FILE_STRICT},
-		{'I', 0, "data-bytes",			NULL, SOURCE_ENV},
+		{'I', 0, "data-bytes",			NULL, SOURCE_FILE_STRICT},
-		{'u', 0, "block-size",			NULL, SOURCE_ENV},
+		{'u', 0, "block-size",			NULL, SOURCE_FILE_STRICT},
-		{'u', 0, "xlog-block-size",		NULL, SOURCE_ENV},
+		{'u', 0, "xlog-block-size",		NULL, SOURCE_FILE_STRICT},
-		{'u', 0, "checksum_version",	NULL, SOURCE_ENV},
+		{'u', 0, "checksum_version",	NULL, SOURCE_FILE_STRICT},
-		{'u', 0, "stream",				NULL, SOURCE_ENV},
+		{'u', 0, "stream",				NULL, SOURCE_FILE_STRICT},
-		{'s', 0, "status",				NULL, SOURCE_ENV},
+		{'s', 0, "status",				NULL, SOURCE_FILE_STRICT},
-		{'s', 0, "parent_backup",		NULL, SOURCE_ENV},
+		{'s', 0, "parent_backup",		NULL, SOURCE_FILE_STRICT},
 		{0}
 	};
 
@@ -364,7 +376,7 @@ catalog_read_ini(const char *path)
 		return NULL;
 
 	backup = pgut_new(pgBackup);
-	catalog_init_config(backup);
+	init_backup(backup);
 
 	i = 0;
 	options[i++].var = &backup_mode;
@@ -516,7 +528,7 @@ pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subd
 }
 
 void
-catalog_init_config(pgBackup *backup)
+init_backup(pgBackup *backup)
 {
 	backup->backup_mode = BACKUP_MODE_INVALID;
 	backup->status = BACKUP_STATUS_INVALID;

delete.c

@@ -10,10 +10,12 @@
 #include "pg_probackup.h"
 
 #include <dirent.h>
+#include <time.h>
 #include <unistd.h>
 
 static int pgBackupDeleteFiles(pgBackup *backup);
-int do_deletewal(time_t backup_id, bool strict);
+static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
+							bool delete_all);
 
 int
 do_delete(time_t backup_id)
@@ -29,12 +31,12 @@ do_delete(time_t backup_id)
 		elog(ERROR, "required backup ID not specified");
 
 	/* Lock backup catalog */
-	ret = catalog_lock();
+	ret = catalog_lock(false);
 	if (ret == -1)
 		elog(ERROR, "can't lock backup catalog.");
 	else if (ret == 1)
 		elog(ERROR,
-			"another pg_probackup is running, stop delete.");
+			 "another pg_probackup is running, stop delete.");
 
 	/* Get complete list of backups */
 	backup_list = catalog_get_backup_list(0);
@@ -88,37 +90,40 @@ found_backup:
 	return 0;
 }
 
-int do_deletewal(time_t backup_id, bool strict)
+/*
+ * Delete in archive WAL segments that are not needed anymore. The oldest
+ * segment to be kept is the first segment that the oldest full backup
+ * found around needs to keep.
+ */
+int
+do_deletewal(time_t backup_id, bool strict)
 {
-	int			i;
+	size_t		i;
 	int			ret;
 	parray		*backup_list;
 	XLogRecPtr	oldest_lsn = InvalidXLogRecPtr;
 	TimeLineID	oldest_tli;
-	pgBackup	*last_backup;
 	bool		backup_found = false;
 
-	/*
-	 * Delete in archive WAL segments that are not needed anymore. The oldest
-	 * segment to be kept is the first segment that the oldest full backup
-	 * found around needs to keep.
-	 */
 	/* Lock backup catalog */
-	ret = catalog_lock();
+	ret = catalog_lock(false);
 	if (ret == -1)
 		elog(ERROR, "can't lock backup catalog.");
 	else if (ret == 1)
 		elog(ERROR,
-			"another pg_probackup is running, stop delete.");
+			 "another pg_probackup is running, stop delete.");
 
+	/* Find oldest LSN, used by backups */
 	backup_list = catalog_get_backup_list(0);
 	for (i = 0; i < parray_num(backup_list); i++)
 	{
-		last_backup = (pgBackup *) parray_get(backup_list, i);
+		pgBackup   *last_backup = (pgBackup *) parray_get(backup_list, i);
+
 		if (last_backup->status == BACKUP_STATUS_OK)
 		{
 			oldest_lsn = last_backup->start_lsn;
 			oldest_tli = last_backup->tli;
+
 			if (strict && backup_id != 0 && backup_id >= last_backup->start_time)
 			{
 				backup_found = true;
@@ -126,196 +131,98 @@ int do_deletewal(time_t backup_id, bool strict)
 			}
 		}
 	}
+
 	if (strict && backup_id != 0 && backup_found == false)
 		elog(ERROR, "not found backup for deletwal command");
+
 	catalog_unlock();
 	parray_walk(backup_list, pgBackupFree);
 	parray_free(backup_list);
 
-	if (!XLogRecPtrIsInvalid(oldest_lsn))
+	delete_walfiles(oldest_lsn, oldest_tli, false);
-	{
-		XLogSegNo   targetSegNo;
-		char		oldestSegmentNeeded[MAXFNAMELEN];
-		DIR		   *arcdir;
-		struct dirent *arcde;
-		char		wal_file[MAXPGPATH];
-		char		max_wal_file[MAXPGPATH];
-		char		min_wal_file[MAXPGPATH];
-		int			rc;
-
-		max_wal_file[0] = '\0';
-		min_wal_file[0] = '\0';
-		XLByteToSeg(oldest_lsn, targetSegNo);
-		XLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo);
-		elog(LOG, "Removing segments older than %s", oldestSegmentNeeded);
-
-		/*
-		 * Now is time to do the actual work and to remove all the segments
-		 * not needed anymore.
-		 */
-		if ((arcdir = opendir(arclog_path)) != NULL)
-		{
-			while (errno = 0, (arcde = readdir(arcdir)) != NULL)
-			{
-				/*
-				 * We ignore the timeline part of the XLOG segment identifiers in
-				 * deciding whether a segment is still needed.  This ensures that
-				 * we won't prematurely remove a segment from a parent timeline.
-				 * We could probably be a little more proactive about removing
-				 * segments of non-parent timelines, but that would be a whole lot
-				 * more complicated.
-				 *
-				 * We use the alphanumeric sorting property of the filenames to
-				 * decide which ones are earlier than the exclusiveCleanupFileName
-				 * file. Note that this means files are not removed in the order
-				 * they were originally written, in case this worries you.
-				 */
-				if ((IsXLogFileName(arcde->d_name) ||
-					 IsPartialXLogFileName(arcde->d_name)) &&
-					strcmp(arcde->d_name + 8, oldestSegmentNeeded + 8) < 0)
-				{
-					/*
-					 * Use the original file name again now, including any
-					 * extension that might have been chopped off before testing
-					 * the sequence.
-					 */
-					snprintf(wal_file, MAXPGPATH, "%s/%s",
-							 arclog_path, arcde->d_name);
-
-					rc = unlink(wal_file);
-					if (rc != 0)
-					{
-						elog(WARNING, "could not remove file \"%s\": %s",
-							 wal_file, strerror(errno));
-						break;
-					}
-					if (verbose)
-						elog(LOG, "removed WAL segment \"%s\"", wal_file);
-
-					if (max_wal_file[0] == '\0' ||
-						strcmp(max_wal_file + 8, arcde->d_name + 8) < 0)
-					{
-						strcpy(max_wal_file, arcde->d_name);
-					}
-
-					if (min_wal_file[0] == '\0' ||
-						strcmp(min_wal_file + 8, arcde->d_name + 8) > 0)
-					{
-						strcpy(min_wal_file, arcde->d_name);
-					}
-				}
-			}
-
-			if (!verbose && min_wal_file[0] != '\0')
-				elog(NOTICE, "removed min WAL segment \"%s\"", min_wal_file);
-			if (!verbose && max_wal_file[0] != '\0')
-				elog(NOTICE, "removed max WAL segment \"%s\"", max_wal_file);
-
-			if (errno)
-				elog(WARNING, "could not read archive location \"%s\": %s",
-					 arclog_path, strerror(errno));
-			if (closedir(arcdir))
-				elog(WARNING, "could not close archive location \"%s\": %s",
-					 arclog_path, strerror(errno));
-		}
-		else
-			elog(WARNING, "could not open archive location \"%s\": %s",
-				 arclog_path, strerror(errno));
-	}
 
 	return 0;
 }
 
 /*
- * Delete backups that are older than KEEP_xxx_DAYS and have more generations
+ * Remove backups by retention policy. Retention policy is configured by
- * than KEEP_xxx_FILES.
+ * retention_redundancy and retention_window variables.
  */
-void
+int
-pgBackupDelete(int keep_generations, int keep_days)
+do_retention_purge(void)
 {
-	int		i;
+	parray	   *backup_list;
-	parray *backup_list;
+	uint32		backup_num;
-	int		backup_num;
+	size_t		i;
-	time_t	days_threshold = current.start_time - (keep_days * 60 * 60 * 24);
+	time_t		days_threshold = time(NULL) - (retention_window * 60 * 60 * 24);
+	XLogRecPtr	oldest_lsn = InvalidXLogRecPtr;
+	TimeLineID	oldest_tli;
+	int			ret;
 
-	if (verbose)
+	if (retention_redundancy > 0)
-	{
+		elog(LOG, "REDUNDANCY=%u", retention_redundancy);
-		char generations_str[100];
+	if (retention_window > 0)
-		char days_str[100];
+		elog(LOG, "WINDOW=%u", retention_window);
 
-		if (keep_generations == KEEP_INFINITE)
+	if (retention_redundancy == 0 && retention_window == 0)
-			strncpy(generations_str, "INFINITE",
+		elog(ERROR, "retention policy is not set");
-					lengthof(generations_str));
-		else
-			snprintf(generations_str, lengthof(generations_str),
-					"%d", keep_generations);
 
-		if (keep_days == KEEP_INFINITE)
+	/* Lock backup catalog */
-			strncpy(days_str, "INFINITE", lengthof(days_str));
+	ret = catalog_lock(false);
-		else
+	if (ret == 1)
-			snprintf(days_str, lengthof(days_str), "%d", keep_days);
+		elog(ERROR,
-
+			 "cannot lock backup catalog, another pg_probackup is running");
-		elog(LOG, "deleted old backups (generations=%s, days=%s)",
-			 generations_str, days_str);
-	}
-
-	/* Leave if an infinite generation of backups is kept */
-	if (keep_generations == KEEP_INFINITE && keep_days == KEEP_INFINITE)
-	{
-		elog(LOG, "%s() infinite", __FUNCTION__);
-		return;
-	}
 
 	/* Get a complete list of backups. */
 	backup_list = catalog_get_backup_list(0);
+	if (parray_num(backup_list) == 0)
+	{
+		elog(INFO, "backup list is empty");
+		elog(INFO, "exit");
+		catalog_unlock();
+		return 0;
+	}
 
 	/* Find target backups to be deleted */
 	backup_num = 0;
 	for (i = 0; i < parray_num(backup_list); i++)
 	{
 		pgBackup   *backup = (pgBackup *) parray_get(backup_list, i);
-		int			backup_num_evaluate = backup_num;
+		uint32		backup_num_evaluate = backup_num;
-
-		elog(LOG, "%s() %lu", __FUNCTION__, backup->start_time);
 
 		/*
 		 * When a validate full backup was found, we can delete the
 		 * backup that is older than it using the number of generations.
 		 */
-		if (backup->backup_mode == BACKUP_MODE_FULL &&
+		if (backup->backup_mode == BACKUP_MODE_FULL)
-			backup->status == BACKUP_STATUS_OK)
 			backup_num++;
 
 		/* Evaluate if this backup is eligible for removal */
-		if (backup_num_evaluate + 1 <= keep_generations &&
+		if (backup_num_evaluate + 1 <= retention_redundancy ||
-			keep_generations != KEEP_INFINITE)
+			(retention_window > 0 && backup->start_time >= days_threshold))
 		{
-			/* Do not include the latest full backup in this count */
+			/* Save LSN and Timeline to remove unnecessary WAL segments */
-			elog(LOG, "%s() backup are only %d", __FUNCTION__, backup_num);
+			oldest_lsn = backup->start_lsn;
-			continue;
+			oldest_tli = backup->tli;
-		}
-		else if (backup->start_time >= days_threshold &&
-				 keep_days != KEEP_INFINITE)
-		{
-			/*
-			 * If the start time of the backup is older than the threshold and
-			 * there are enough generations of full backups, delete the backup.
-			 */
-			elog(LOG, "%s() %lu is not older than %lu", __FUNCTION__,
-				backup->start_time, days_threshold);
 			continue;
 		}
 
-		elog(LOG, "%s() %lu is older than %lu", __FUNCTION__,
+		/* Delete backup and update status to DELETED */
-			backup->start_time, days_threshold);
-
-		/* delete backup and update status to DELETED */
 		pgBackupDeleteFiles(backup);
 	}
 
-	/* cleanup */
+	/* Purge WAL files */
+	delete_walfiles(oldest_lsn, oldest_tli, true);
+
+	/* Cleanup */
 	parray_walk(backup_list, pgBackupFree);
 	parray_free(backup_list);
+
+	catalog_unlock();
+
+	elog(INFO, "purging is finished");
+
+	return 0;
 }
 
 /*
@@ -325,10 +232,10 @@ pgBackupDelete(int keep_generations, int keep_days)
 static int
 pgBackupDeleteFiles(pgBackup *backup)
 {
-	int		i;
+	size_t		i;
-	char	path[MAXPGPATH];
+	char		path[MAXPGPATH];
-	char	timestamp[20];
+	char		timestamp[20];
-	parray *files;
+	parray	   *files;
 
 	/*
 	 * If the backup was deleted already, there is nothing to do.
@@ -340,7 +247,6 @@ pgBackupDeleteFiles(pgBackup *backup)
 
 	elog(INFO, "delete: %s %s", base36enc(backup->start_time), timestamp);
 
-
 	/*
 	 * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which
 	 * the error occurs before deleting all backup files.
@@ -363,7 +269,7 @@ pgBackupDeleteFiles(pgBackup *backup)
 		pgFile *file = (pgFile *) parray_get(files, i);
 
 		/* print progress */
-		elog(LOG, "delete file(%d/%lu) \"%s\"", i + 1,
+		elog(LOG, "delete file(%zd/%lu) \"%s\"", i + 1,
 				(unsigned long) parray_num(files), file->path);
 
 		/* skip actual deletion in check mode */
@@ -375,6 +281,7 @@ pgBackupDeleteFiles(pgBackup *backup)
 					strerror(errno));
 				parray_walk(files, pgFileFree);
 				parray_free(files);
+
 				return 1;
 			}
 		}
@@ -385,3 +292,112 @@ pgBackupDeleteFiles(pgBackup *backup)
 
 	return 0;
 }
+
+/*
+ * Delete WAL segments up to oldest_lsn.
+ *
+ * If oldest_lsn is invalid function exists. But if delete_all is true then
+ * WAL segements will be deleted anyway.
+ */
+static void
+delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all)
+{
+	XLogSegNo   targetSegNo;
+	char		oldestSegmentNeeded[MAXFNAMELEN];
+	DIR		   *arcdir;
+	struct dirent *arcde;
+	char		wal_file[MAXPGPATH];
+	char		max_wal_file[MAXPGPATH];
+	char		min_wal_file[MAXPGPATH];
+	int			rc;
+
+	if (XLogRecPtrIsInvalid(oldest_lsn) && !delete_all)
+		return;
+
+	max_wal_file[0] = '\0';
+	min_wal_file[0] = '\0';
+
+	if (!XLogRecPtrIsInvalid(oldest_lsn))
+	{
+		XLByteToSeg(oldest_lsn, targetSegNo);
+		XLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo);
+
+		elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded);
+	}
+	else
+		elog(LOG, "removing all WAL segments");
+
+	/*
+	 * Now it is time to do the actual work and to remove all the segments
+	 * not needed anymore.
+	 */
+	if ((arcdir = opendir(arclog_path)) != NULL)
+	{
+		while (errno = 0, (arcde = readdir(arcdir)) != NULL)
+		{
+			/*
+			 * We ignore the timeline part of the XLOG segment identifiers in
+			 * deciding whether a segment is still needed.  This ensures that
+			 * we won't prematurely remove a segment from a parent timeline.
+			 * We could probably be a little more proactive about removing
+			 * segments of non-parent timelines, but that would be a whole lot
+			 * more complicated.
+			 *
+			 * We use the alphanumeric sorting property of the filenames to
+			 * decide which ones are earlier than the exclusiveCleanupFileName
+			 * file. Note that this means files are not removed in the order
+			 * they were originally written, in case this worries you.
+			 */
+			if (IsXLogFileName(arcde->d_name) ||
+				IsPartialXLogFileName(arcde->d_name) ||
+				IsBackupHistoryFileName(arcde->d_name))
+			{
+				if (XLogRecPtrIsInvalid(oldest_lsn) ||
+					strncmp(arcde->d_name, oldestSegmentNeeded,
+							XLOG_FNAME_LEN) < 0)
+				{
+					/*
+					 * Use the original file name again now, including any
+					 * extension that might have been chopped off before testing
+					 * the sequence.
+					 */
+					snprintf(wal_file, MAXPGPATH, "%s/%s",
+							 arclog_path, arcde->d_name);
+
+					rc = unlink(wal_file);
+					if (rc != 0)
+					{
+						elog(WARNING, "could not remove file \"%s\": %s",
+							 wal_file, strerror(errno));
+						break;
+					}
+					if (verbose)
+						elog(LOG, "removed WAL segment \"%s\"", wal_file);
+
+					if (max_wal_file[0] == '\0' ||
+						strcmp(max_wal_file + 8, arcde->d_name + 8) < 0)
+						strcpy(max_wal_file, arcde->d_name);
+
+					if (min_wal_file[0] == '\0' ||
+						strcmp(min_wal_file + 8, arcde->d_name + 8) > 0)
+						strcpy(min_wal_file, arcde->d_name);
+				}
+			}
+		}
+
+		if (!verbose && min_wal_file[0] != '\0')
+			elog(INFO, "removed min WAL segment \"%s\"", min_wal_file);
+		if (!verbose && max_wal_file[0] != '\0')
+			elog(INFO, "removed max WAL segment \"%s\"", max_wal_file);
+
+		if (errno)
+			elog(WARNING, "could not read archive location \"%s\": %s",
+				 arclog_path, strerror(errno));
+		if (closedir(arcdir))
+			elog(WARNING, "could not close archive location \"%s\": %s",
+				 arclog_path, strerror(errno));
+	}
+	else
+		elog(WARNING, "could not open archive location \"%s\": %s",
+			 arclog_path, strerror(errno));
+}

doc/pg_probackup.md

@@ -11,6 +11,7 @@ pg_probackup [option...] validate backup_ID
 pg_probackup [option...] show    [backup_ID]
 pg_probackup [option...] delete   backup_ID
 pg_probackup [option...] delwal  [backup_ID]
+pg_probackup [option...] retention show|purge
 ```
 
 ## Description
@@ -294,6 +295,34 @@ The same backup directory can be used for pg\_probackup on both servers, primary
 
 A backup can be used to restore primary database server as well as standby. It depends on the server on which pg\_probackup is executed with restore command. Note that recovered PostgreSQL will always run as primary server if started right after the pg\_probackup. To run it as standby, edit recovery.conf file created by pg\_probackup: at least delete every parameter that specify recovery target (recovery\_target, recovery\_target\_time, and recovery\_target\_xid), change target timeline to 'latest', and add standby\_mode = 'on'. Probably primary\_conninfo should be added too for streaming replication, and hot\_standby = 'on' in database configuration parameters for hot standby mode.
 
+### Backup Retention Policy
+
+It is possible to configure the backup retention policy. The retention policy
+specifies specifies which backups must be kept to meet data recoverability
+requirements. The policy can be configured using two parameters: redundancy and
+window.
+
+Redundancy parameter specifies how many full backups purge command should keep.
+For example, you make a full backup on Sunday and an incremental backup every day.
+If redundancy is 1, then this backup will become obsolete on next Sunday and will
+be deleted with all its incremental backups when next full backup will be created.
+
+Window parameter specifies the number of days of data recoverability. If window
+is 14, then all full backups older than 14 days will be deleted with their
+incremental backups.
+
+This parameters can be used together. Backups are obsolete if they don't
+meet both parameters. For example, you have retention is 1, window is 14 and
+two full backups are made 3 and 7 days ago. In this situation both backups aren't
+obsolete and will be kept 14 days.
+
+To delete obsolete backups execute the following command:
+```
+pg_probackup retention purge
+```
+Redundancy and window parameters values will be taken from command line options
+or from pg_probackup.conf configuration file.
+
 ## Additional Features
 
 ### Parallel Execution
@@ -319,11 +348,11 @@ Pages are packed before going to backup, leaving unused parts of pages behind (s
 
 Whether page checksums are enabled or not, pg\_probackup calculates checksums for each file in a backup. Checksums are checked immediately after backup is taken and right before restore, to timely detect possible backup corruptions.
 
-##Options
+## Options
 
 Options for pg\_probackup utility can be specified in command line (such options are shown below starting from either one or two minus signs). If not given in command line, values for some options are derived from environmental variables (names of environmental variables are in uppercase). Otherwise values for some options are taken from pg\_probackup.conf configuration file, located in the backup directory (such option names are in lowercase).
 
-Common options:
+### Common options:
 
 -B _directory_  
 --backup-path=_directory_  
@@ -365,7 +394,7 @@ Show quick help on command line options.
 
 Show version information.
 
-Backup options:
+### Backup options:
 
 -b _mode_  
 --backup-mode=_mode_  
@@ -430,7 +459,7 @@ Never issue a password prompt. If the server requires password authentication an
 
 Force pg\_probackup to prompt for a password before connecting to a database.
 
-Restore options:
+### Restore options:
 
 --time
 
@@ -448,12 +477,22 @@ Specifies whether to stop just after the specified recovery target (true), or ju
 
 Specifies recovering into a particular timeline.
 
-Delete options:
+### Delete options:
 
 --wal
 
 Delete WAL files that are no longer necessary to restore from any of existing backups.
 
+### Retention policy options:
+
+--redundancy
+
+Specifies how many full backups purge command should keep.
+
+--window
+
+Specifies the number of days of recoverability.
+
 ## Restrictions
 
 Currently pg\_probackup has the following restrictions:

fetch.c

@@ -3,7 +3,7 @@
  * fetch.c
  *    Functions for fetching files from PostgreSQL data directory
  *
- * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */

init.c

@@ -12,9 +12,6 @@
 #include <unistd.h>
 #include <dirent.h>
 
-static void parse_postgresql_conf(const char *path, char **log_directory,
-								  char **archive_command);
-
 /*
  * selects function for scandir.
  */
@@ -31,8 +28,6 @@ do_init(void)
 {
 	char	path[MAXPGPATH];
 	char	arclog_path_dir[MAXPGPATH];
-	char   *log_directory = NULL;
-	char   *archive_command = NULL;
 	FILE   *fp;
 	uint64 _system_identifier;
 
@@ -58,47 +53,20 @@ do_init(void)
 	join_path_components(path, backup_path, BACKUPS_DIR);
 	dir_create_dir(path, DIR_PERMISSION);
 
-	/* read postgresql.conf */
+	/* Create "wal" directory */
-	if (pgdata)
+	join_path_components(arclog_path_dir, backup_path, "wal");
-	{
+	dir_create_dir(arclog_path_dir, DIR_PERMISSION);
-		join_path_components(path, pgdata, "postgresql.conf");
-		parse_postgresql_conf(path, &log_directory, &archive_command);
-	}
 
 	_system_identifier = get_system_identifier(false);
 	/* create pg_probackup.conf */
-	join_path_components(path, backup_path, PG_RMAN_INI_FILE);
+	join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE);
 	fp = fopen(path, "wt");
 	if (fp == NULL)
 		elog(ERROR, "cannot create pg_probackup.conf: %s", strerror(errno));
 
-	join_path_components(arclog_path_dir, backup_path, "wal");
-	dir_create_dir(arclog_path_dir, DIR_PERMISSION);
-
 	fprintf(fp, "system-identifier = %li\n", _system_identifier);
 	fprintf(fp, "\n");
 	fclose(fp);
 
-	free(archive_command);
-	free(log_directory);
-
 	return 0;
 }
-
-static void
-parse_postgresql_conf(const char *path,
-					  char **log_directory,
-					  char **archive_command)
-{
-	pgut_option options[] =
-	{
-		{ 's', 0, "log_directory"		, NULL, SOURCE_ENV },
-		{ 's', 0, "archive_command"		, NULL, SOURCE_ENV },
-		{ 0 }
-	};
-
-	options[0].var = log_directory;
-	options[1].var = archive_command;
-
-	pgut_readopt(path, options, LOG);	/* ignore unknown options */
-}

pg_probackup.c

@@ -32,15 +32,12 @@ pgBackup	current;
 
 /* backup configuration */
 static bool		smooth_checkpoint;
-static int		keep_data_generations = KEEP_INFINITE;
-static int		keep_data_days = KEEP_INFINITE;
 int				num_threads = 1;
 bool			stream_wal = false;
 bool			from_replica = false;
 static bool		backup_logs = false;
 bool			progress = false;
 bool			delete_wal = false;
-uint64			system_identifier = 0;
 
 /* restore configuration */
 static char		   *target_time;
@@ -48,26 +45,30 @@ static char		   *target_xid;
 static char		   *target_inclusive;
 static TimeLineID	target_tli;
 
+uint64			system_identifier = 0;
+
+/* retention configuration */
+uint32			retention_redundancy = 0;
+uint32			retention_window = 0;
+
 static void opt_backup_mode(pgut_option *opt, const char *arg);
 
 static pgut_option options[] =
 {
 	/* directory options */
-	{ 's', 'D', "pgdata",				&pgdata,		SOURCE_ENV },
+	{ 's', 'D', "pgdata",				&pgdata,		SOURCE_CMDLINE },
-	{ 's', 'B', "backup-path",			&backup_path,	SOURCE_ENV },
+	{ 's', 'B', "backup-path",			&backup_path,	SOURCE_CMDLINE },
 	/* common options */
 /*	{ 'b', 'c', "check",				&check },*/
-	{ 'i', 'j', "threads",				&num_threads },
+	{ 'u', 'j', "threads",				&num_threads,	SOURCE_CMDLINE },
-	{ 'b', 8, "stream",					&stream_wal },
+	{ 'b', 8, "stream",					&stream_wal,	SOURCE_CMDLINE },
-	{ 'b', 11, "progress",				&progress },
+	{ 'b', 11, "progress",				&progress,		SOURCE_CMDLINE },
 	/* backup options */
-	{ 'b', 10, "backup-pg-log",			&backup_logs },
+	{ 'b', 10, "backup-pg-log",			&backup_logs,	SOURCE_CMDLINE },
-	{ 'f', 'b', "backup-mode",			opt_backup_mode,		SOURCE_ENV },
+	{ 'f', 'b', "backup-mode",			opt_backup_mode,		SOURCE_CMDLINE },
-	{ 'b', 'C', "smooth-checkpoint",	&smooth_checkpoint,		SOURCE_ENV },
+	{ 'b', 'C', "smooth-checkpoint",	&smooth_checkpoint,		SOURCE_CMDLINE },
 	{ 's', 'S', "slot",					&replication_slot,		SOURCE_CMDLINE },
 	/* options with only long name (keep-xxx) */
-/*	{ 'i',  1, "keep-data-generations", &keep_data_generations, SOURCE_ENV },
-	{ 'i',  2, "keep-data-days",		&keep_data_days,		SOURCE_ENV },*/
 	/* restore options */
 	{ 's',  3, "time",					&target_time,		SOURCE_CMDLINE },
 	{ 's',  4, "xid",					&target_xid,		SOURCE_CMDLINE },
@@ -75,8 +76,11 @@ static pgut_option options[] =
 	{ 'u',  6, "timeline",				&target_tli,		SOURCE_CMDLINE },
 	/* delete options */
 	{ 'b', 12, "wal",					&delete_wal },
+	/* retention options */
+	{ 'u', 13, "redundancy",			&retention_redundancy,	SOURCE_CMDLINE },
+	{ 'u', 14, "window",				&retention_window,		SOURCE_CMDLINE },
 	/* other */
-	{ 'U', 13, "system-identifier",		&system_identifier,	SOURCE_FILE },
+	{ 'U', 15, "system-identifier",		&system_identifier,		SOURCE_FILE_STRICT },
 	{ 0 }
 };
 
@@ -86,7 +90,8 @@ static pgut_option options[] =
 int
 main(int argc, char *argv[])
 {
-	const char	   *cmd = NULL;
+	const char	   *cmd = NULL,
+				   *subcmd = NULL;
 	const char	   *backup_id_string = NULL;
 	time_t			backup_id = 0;
 	int				i;
@@ -95,7 +100,7 @@ main(int argc, char *argv[])
 	setvbuf(stdout, 0, _IONBF, 0);	/* TODO: remove this */
 
 	/* initialize configuration */
-	catalog_init_config(&current);
+	init_backup(&current);
 
 	/* overwrite configuration with command line arguments */
 	i = pgut_getopt(argc, argv, options);
@@ -103,15 +108,15 @@ main(int argc, char *argv[])
 	for (; i < argc; i++)
 	{
 		if (cmd == NULL)
-		{
 			cmd = argv[i];
-			if(strcmp(cmd, "show") != 0 &&
+		else if (strcmp(cmd, "retention") == 0)
-			   strcmp(cmd, "validate") != 0 &&
+			subcmd = argv[i];
-			   strcmp(cmd, "delete") != 0 &&
+		else if (backup_id_string == NULL &&
-			   strcmp(cmd, "restore") != 0 &&
+				 (strcmp(cmd, "show") == 0 ||
-			   strcmp(cmd, "delwal") != 0)
+				 strcmp(cmd, "validate") == 0 ||
-				break;
+				 strcmp(cmd, "delete") == 0 ||
-		} else if (backup_id_string == NULL)
+				 strcmp(cmd, "restore") == 0 ||
+				 strcmp(cmd, "delwal") == 0))
 			backup_id_string = argv[i];
 		else
 			elog(ERROR, "too many arguments");
@@ -132,38 +137,36 @@ main(int argc, char *argv[])
 		}
 	}
 
-	/* Read default configuration from file. */
+	/* BACKUP_PATH is always required */
-	if (backup_path)
+	if (backup_path == NULL)
+		elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)");
+	else
 	{
-		char	path[MAXPGPATH];
+		char		path[MAXPGPATH];
 		/* Check if backup_path is directory. */
 		struct stat stat_buf;
-		int rc = stat(backup_path, &stat_buf);
+		int			rc = stat(backup_path, &stat_buf);
 
 		/* If rc == -1,  there is no file or directory. So it's OK. */
 		if (rc != -1 && !S_ISDIR(stat_buf.st_mode))
 			elog(ERROR, "-B, --backup-path must be a path to directory");
 
-		join_path_components(path, backup_path, PG_RMAN_INI_FILE);
+		join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE);
 		pgut_readopt(path, options, ERROR);
-
-		/* setup stream options */
-		if (pgut_dbname != NULL)
-			dbname = pstrdup(pgut_dbname);
-		if (host != NULL)
-			dbhost = pstrdup(host);
-		if (port != NULL)
-			dbport = pstrdup(port);
-		if (username != NULL)
-			dbuser = pstrdup(username);
 	}
 
-	/* BACKUP_PATH is always required */
+	/* setup stream options */
-	if (backup_path == NULL)
+	if (pgut_dbname != NULL)
-		elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)");
+		dbname = pstrdup(pgut_dbname);
+	if (host != NULL)
+		dbhost = pstrdup(host);
+	if (port != NULL)
+		dbport = pstrdup(port);
+	if (username != NULL)
+		dbuser = pstrdup(username);
 
 	/* path must be absolute */
-	if (backup_path != NULL && !is_absolute_path(backup_path))
+	if (!is_absolute_path(backup_path))
 		elog(ERROR, "-B, --backup-path must be an absolute path");
 	if (pgdata != NULL && !is_absolute_path(pgdata))
 		elog(ERROR, "-D, --pgdata must be an absolute path");
@@ -189,20 +192,10 @@ main(int argc, char *argv[])
 		return do_init();
 	else if (pg_strcasecmp(cmd, "backup") == 0)
 	{
-		pgBackupOption bkupopt;
+		int			res;
-		int res;
-		uint64 _system_identifier;
-		bkupopt.smooth_checkpoint = smooth_checkpoint;
-		bkupopt.keep_data_generations = keep_data_generations;
-		bkupopt.keep_data_days = keep_data_days;
-
-		_system_identifier = get_system_identifier(true);
-		if (_system_identifier != system_identifier)
-			elog(ERROR, "Backup directory was initialized for system id = %ld, but target system id = %ld",
-				 system_identifier, _system_identifier);
 
 		/* Do the backup */
-		res = do_backup(bkupopt);
+		res = do_backup(smooth_checkpoint);
 		if (res != 0)
 			return res;
 
@@ -230,6 +223,15 @@ main(int argc, char *argv[])
 		return do_delete(backup_id);
 	else if (pg_strcasecmp(cmd, "delwal") == 0)
 		return do_deletewal(backup_id, true);
+	else if (pg_strcasecmp(cmd, "retention") == 0)
+	{
+		if (subcmd == NULL)
+			elog(ERROR, "you must specify retention command");
+		else if (pg_strcasecmp(subcmd, "show") == 0)
+			return do_retention_show();
+		else if (pg_strcasecmp(subcmd, "purge") == 0)
+			return do_retention_purge();
+	}
 	else
 		elog(ERROR, "invalid command \"%s\"", cmd);
 
@@ -243,11 +245,12 @@ pgut_help(bool details)
 	printf(_("Usage:\n"));
 	printf(_("  %s [option...] init\n"), PROGRAM_NAME);
 	printf(_("  %s [option...] backup\n"), PROGRAM_NAME);
-	printf(_("  %s [option...] restore\n"), PROGRAM_NAME);
+	printf(_("  %s [option...] restore [backup-ID]\n"), PROGRAM_NAME);
 	printf(_("  %s [option...] show [backup-ID]\n"), PROGRAM_NAME);
 	printf(_("  %s [option...] validate backup-ID\n"), PROGRAM_NAME);
 	printf(_("  %s [option...] delete backup-ID\n"), PROGRAM_NAME);
 	printf(_("  %s [option...] delwal [backup-ID]\n"), PROGRAM_NAME);
+	printf(_("  %s [option...] retention show|purge\n"), PROGRAM_NAME);
 
 	if (!details)
 		return;
@@ -260,8 +263,6 @@ pgut_help(bool details)
 	printf(_("  -b, --backup-mode=MODE    backup mode (full, page, ptrack)\n"));
 	printf(_("  -C, --smooth-checkpoint   do smooth checkpoint before backup\n"));
 	printf(_("      --stream              stream the transaction log and include it in the backup\n"));
-	/*printf(_("  --keep-data-generations=N keep GENERATION of full data backup\n"));
-	printf(_("  --keep-data-days=DAY      keep enough data backup to recover to DAY days age\n"));*/
 	printf(_("  -S, --slot=SLOTNAME       replication slot to use\n"));
 	printf(_("      --backup-pg-log       backup of pg_log directory\n"));
 	printf(_("  -j, --threads=NUM         number of parallel threads\n"));
@@ -275,6 +276,9 @@ pgut_help(bool details)
 	printf(_("      --progress            show progress\n"));
 	printf(_("\nDelete options:\n"));
 	printf(_("      --wal                 remove unnecessary wal files\n"));
+	printf(_("\nRetention options:\n"));
+	printf(_("      --redundancy          specifies how many full backups purge command should keep\n"));
+	printf(_("      --window              specifies the number of days of recoverability\n"));
 }
 
 static void

pg_probackup.h

@@ -34,8 +34,8 @@
 #define BACKUPS_DIR				"backups"
 #define PG_XLOG_DIR				"pg_xlog"
 #define PG_TBLSPC_DIR			"pg_tblspc"
-#define BACKUP_INI_FILE			"backup.conf"
+#define BACKUP_CONF_FILE		"backup.conf"
-#define PG_RMAN_INI_FILE		"pg_probackup.conf"
+#define BACKUP_CATALOG_CONF_FILE	"pg_probackup.conf"
 #define MKDIRS_SH_FILE			"mkdirs.sh"
 #define DATABASE_FILE_LIST		"file_database.txt"
 #define PG_BACKUP_LABEL_FILE	"backup_label"
@@ -141,14 +141,6 @@ typedef struct pgBackup
 	time_t			parent_backup;
 } pgBackup;
 
-typedef struct pgBackupOption
-{
-	bool smooth_checkpoint;
-	int  keep_data_generations;
-	int  keep_data_days;
-} pgBackupOption;
-
-
 /* special values of pgBackup */
 #define KEEP_INFINITE			(INT_MAX)
 #define BYTES_INVALID			(-1)
@@ -210,10 +202,15 @@ extern bool stream_wal;
 extern bool from_replica;
 extern bool progress;
 extern bool delete_wal;
+
 extern uint64 system_identifier;
 
+/* retention configuration */
+extern uint32 retention_redundancy;
+extern uint32 retention_window;
+
 /* in backup.c */
-extern int do_backup(pgBackupOption bkupopt);
+extern int do_backup(bool smooth_checkpoint);
 extern BackupMode parse_backup_mode(const char *value);
 extern void check_server_version(void);
 extern bool fileExists(const char *path);
@@ -243,11 +240,12 @@ extern int do_init(void);
 
 /* in show.c */
 extern int do_show(time_t backup_id);
+extern int do_retention_show(void);
 
 /* in delete.c */
 extern int do_delete(time_t backup_id);
-extern void pgBackupDelete(int keep_generations, int keep_days);
 extern int do_deletewal(time_t backup_id, bool strict);
+extern int do_retention_purge(void);
 
 /* in fetch.c */
 extern char *slurpFile(const char *datadir,
@@ -266,17 +264,16 @@ extern void pgBackupValidate(pgBackup *backup,
 							 bool size_only,
 							 bool for_get_timeline);
 
-/* in catalog.c */
+extern pgBackup *read_backup(time_t timestamp);
-extern pgBackup *catalog_get_backup(time_t timestamp);
+extern void init_backup(pgBackup *backup);
+
 extern parray *catalog_get_backup_list(time_t backup_id);
 extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
 											  TimeLineID tli);
 
-extern int catalog_lock(void);
+extern int catalog_lock(bool check_catalog);
 extern void catalog_unlock(void);
 
-extern void catalog_init_config(pgBackup *backup);
-
 extern void pgBackupWriteConfigSection(FILE *out, pgBackup *backup);
 extern void pgBackupWriteResultSection(FILE *out, pgBackup *backup);
 extern void pgBackupWriteIni(pgBackup *backup);

pgut/pgut.c

@@ -71,15 +71,15 @@ static const char *get_username(void);
 
 static pgut_option default_options[] =
 {
-	{ 's', 'd', "dbname"	, &pgut_dbname },
+	{ 's', 'd', "dbname"	, &pgut_dbname, SOURCE_CMDLINE },
-	{ 's', 'h', "host"		, &host },
+	{ 's', 'h', "host"		, &host, SOURCE_CMDLINE },
-	{ 's', 'p', "port"		, &port },
+	{ 's', 'p', "port"		, &port, SOURCE_CMDLINE },
-	{ 'b', 'q', "quiet"		, &quiet },
+	{ 'b', 'q', "quiet"		, &quiet, SOURCE_CMDLINE },
-	{ 's', 'U', "username"	, &username },
+	{ 's', 'U', "username"	, &username, SOURCE_CMDLINE },
-	{ 'b', 'v', "verbose"	, &verbose },
+	{ 'b', 'v', "verbose"	, &verbose, SOURCE_CMDLINE },
 #ifndef PGUT_NO_PROMPT
-	{ 'Y', 'w', "no-password"	, &prompt_password },
+	{ 'Y', 'w', "no-password"	, &prompt_password, SOURCE_CMDLINE },
-	{ 'y', 'W', "password"		, &prompt_password },
+	{ 'y', 'W', "password"		, &prompt_password, SOURCE_CMDLINE },
 #endif
 	{ 0 }
 };
@@ -576,8 +576,7 @@ option_from_env(pgut_option options[])
 		const char	   *s;
 		const char	   *value;
 
-		if (opt->source > SOURCE_ENV ||
+		if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV)
-			opt->allowed == SOURCE_DEFAULT || opt->allowed > SOURCE_ENV)
 			continue;
 
 		for (s = opt->lname, j = 0; *s && j < lengthof(name) - 1; s++, j++)
@@ -632,6 +631,9 @@ pgut_getopt(int argc, char **argv, pgut_option options[])
 	while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1)
 	{
 		opt = option_find(c, default_options, options);
+		if (opt->allowed < SOURCE_CMDLINE)
+			elog(ERROR, "option %s cannot be specified in command line",
+				 opt->lname);
 		assign_option(opt, optarg, SOURCE_CMDLINE);
 	}
 
@@ -698,8 +700,8 @@ pgut_readopt(const char *path, pgut_option options[], int elevel)
 
 				if (key_equals(key, opt->lname))
 				{
-					if (opt->allowed == SOURCE_DEFAULT ||
+					if (opt->allowed < SOURCE_FILE &&
-						opt->allowed > SOURCE_FILE)
+						opt->allowed != SOURCE_FILE_STRICT)
 						elog(elevel, "option %s cannot specified in file", opt->lname);
 					else if (opt->source <= SOURCE_FILE)
 						assign_option(opt, value, SOURCE_FILE);

pgut/pgut.h

@@ -40,6 +40,7 @@ typedef enum YesNo
 typedef enum pgut_optsrc
 {
 	SOURCE_DEFAULT,
+	SOURCE_FILE_STRICT,
 	SOURCE_ENV,
 	SOURCE_FILE,
 	SOURCE_CMDLINE,

restore.c

@@ -30,9 +30,6 @@ static void create_recovery_conf(time_t backup_id,
 								 const char *target_inclusive,
 								 TimeLineID target_tli);
 static void print_backup_lsn(const pgBackup *backup);
-static void search_next_wal(const char *path,
-							XLogRecPtr *need_lsn,
-							parray *timelines);
 static void restore_files(void *arg);
 
 
@@ -48,7 +45,6 @@ do_restore(time_t backup_id,
 {
 	int i;
 	int base_index;				/* index of base (full) backup */
-	int last_restored_index;	/* index of last restored database backup */
 	int ret;
 	TimeLineID	cur_tli;
 	TimeLineID	backup_tli;
@@ -60,7 +56,6 @@ do_restore(time_t backup_id,
 	pgBackup *base_backup = NULL;
 	pgBackup *dest_backup = NULL;
 	pgRecoveryTarget *rt = NULL;
-	XLogRecPtr need_lsn;
 	bool backup_id_found = false;
 
 	/* PGDATA and ARCLOG_PATH are always required */
@@ -72,12 +67,12 @@ do_restore(time_t backup_id,
 	elog(LOG, "restore start");
 
 	/* get exclusive lock of backup catalog */
-	ret = catalog_lock();
+	ret = catalog_lock(false);
 	if (ret == -1)
 		elog(ERROR, "cannot lock backup catalog.");
 	else if (ret == 1)
 		elog(ERROR,
-			"another pg_probackup is running, stop restore.");
+			 "another pg_probackup is running, stop restore.");
 
 	/* confirm the PostgreSQL server is not running */
 	if (is_pg_running())
@@ -185,8 +180,6 @@ base_backup_found:
 	/* restore base backup */
 	restore_database(base_backup);
 
-	last_restored_index = base_index;
-
 	/* restore following differential backup */
 	elog(LOG, "searching differential backup...");
 
@@ -220,19 +213,8 @@ base_backup_found:
 
 		print_backup_lsn(backup);
 		restore_database(backup);
-		last_restored_index = i;
 	}
 
-	if (!stream_wal || target_time != NULL || target_xid != NULL)
-		for (i = last_restored_index; i >= 0; i--)
-		{
-			elog(LOG, "searching archived WAL...");
-
-			search_next_wal(arclog_path, &need_lsn, timelines);
-
-			elog(LOG, "all necessary files are found");
-		}
-
 	/* create recovery.conf */
 	if (!stream_wal || target_time != NULL || target_xid != NULL)
 		create_recovery_conf(backup_id, target_time, target_xid, target_inclusive, target_tli);
@@ -713,65 +695,6 @@ print_backup_lsn(const pgBackup *backup)
 		 (uint32) backup->stop_lsn);
 }
 
-static void
-search_next_wal(const char *path, XLogRecPtr *need_lsn, parray *timelines)
-{
-	int		i;
-	int		j;
-	int		count;
-	char	xlogfname[MAXFNAMELEN];
-	char	pre_xlogfname[MAXFNAMELEN];
-	char	xlogpath[MAXPGPATH];
-	struct stat	st;
-
-	count = 0;
-	for (;;)
-	{
-		for (i = 0; i < parray_num(timelines); i++)
-		{
-			pgTimeLine *timeline = (pgTimeLine *) parray_get(timelines, i);
-			XLogSegNo	targetSegNo;
-
-			XLByteToSeg(*need_lsn, targetSegNo);
-			XLogFileName(xlogfname, timeline->tli, targetSegNo);
-			join_path_components(xlogpath, path, xlogfname);
-
-			if (stat(xlogpath, &st) == 0)
-				break;
-		}
-
-		/* not found */
-		if (i == parray_num(timelines))
-		{
-			if (count == 1)
-				elog(LOG, "\n");
-			else if (count > 1)
-				elog(LOG, " - %s", pre_xlogfname);
-
-			return;
-		}
-
-		count++;
-		if (count == 1)
-			elog(LOG, "%s", xlogfname);
-
-		strcpy(pre_xlogfname, xlogfname);
-
-		/* delete old TLI */
-		for (j = i + 1; j < parray_num(timelines); j++)
-			parray_remove(timelines, i + 1);
-		/* XXX: should we add a linebreak when we find a timeline? */
-
-		/*
-		 * Move to next xlog segment. Note that we need to increment
-		 * by XLogSegSize to jump directly to the next WAL segment file
-		 * and as this value is used to generate the WAL file name with
-		 * the current timeline of backup.
-		 */
-		*need_lsn += XLogSegSize;
-	}
-}
-
 pgRecoveryTarget *
 checkIfCreateRecoveryConf(const char *target_time,
                    const char *target_xid,

show.c

@@ -29,7 +29,7 @@ do_show(time_t backup_id)
 	{
 		pgBackup *backup;
 
-		backup = catalog_get_backup(backup_id);
+		backup = read_backup(backup_id);
 		if (backup == NULL)
 		{
 			char timestamp[100];
@@ -62,6 +62,21 @@ do_show(time_t backup_id)
 	return 0;
 }
 
+/*
+ * Show backup catalog retention policy configuration.
+ */
+int
+do_retention_show(void)
+{
+	fprintf(stdout, "# retention policy\n");
+	if (retention_redundancy > 0)
+		fprintf(stdout, "REDUNDANCY=%u\n", retention_redundancy);
+	if (retention_window > 0)
+		fprintf(stdout, "WINDOW=%u\n", retention_window);
+
+	return 0;
+}
+
 static void
 pretty_size(int64 size, char *buf, size_t len)
 {

status.c

@@ -2,7 +2,7 @@
  *
  * status.c
  *
- * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
  *
  * Monitor status of a PostgreSQL server.
  *

tests/__init__.py

@@ -1,6 +1,8 @@
 import unittest
 
-from . import init_test, option_test, show_test, backup_test, delete_test, restore_test, validate_test
+from . import init_test, option_test, show_test, \
+	backup_test, delete_test, restore_test, validate_test, \
+	retention_test
 
 
 def load_tests(loader, tests, pattern):
@@ -12,5 +14,6 @@ def load_tests(loader, tests, pattern):
 	suite.addTests(loader.loadTestsFromModule(delete_test))
 	suite.addTests(loader.loadTestsFromModule(restore_test))
 	suite.addTests(loader.loadTestsFromModule(validate_test))
+	suite.addTests(loader.loadTestsFromModule(retention_test))
 
 	return suite

tests/expected/option_help.out

@@ -3,11 +3,12 @@ pg_probackup manage backup/recovery of PostgreSQL database.
 Usage:
   pg_probackup [option...] init
   pg_probackup [option...] backup
-  pg_probackup [option...] restore
+  pg_probackup [option...] restore [backup-ID]
   pg_probackup [option...] show [backup-ID]
   pg_probackup [option...] validate backup-ID
   pg_probackup [option...] delete backup-ID
   pg_probackup [option...] delwal [backup-ID]
+  pg_probackup [option...] retention show|purge
 
 Common Options:
   -B, --backup-path=PATH    location of the backup storage area
@@ -33,6 +34,10 @@ Restore options:
 Delete options:
       --wal                 remove unnecessary wal files
 
+Retention options:
+      --redundancy          specifies how many full backups purge command should keep
+      --window              specifies the number of days of recoverability
+
 Connection options:
   -d, --dbname=DBNAME       database to connect
   -h, --host=HOSTNAME       database server host or socket directory

tests/pb_lib.py

@@ -187,6 +187,14 @@ class ProbackupTest(object):
 		# print(cmd_list)
 		return self.run_pb(options + cmd_list)
 
+	def retention_purge_pb(self, node, options=[]):
+		cmd_list = [
+			"-B", self.backup_dir(node),
+			"retention", "purge",
+		]
+
+		return self.run_pb(options + cmd_list)
+
 	def get_control_data(self, node):
 		pg_controldata = node.get_bin_path("pg_controldata")
 		out_data = {}

tests/retention_test.py

@@ -0,0 +1,66 @@
+import unittest
+import os
+from datetime import datetime, timedelta
+from os import path
+from .pb_lib import ProbackupTest
+from testgres import stop_all
+
+
+class RetentionTest(ProbackupTest, unittest.TestCase):
+
+	def __init__(self, *args, **kwargs):
+		super(RetentionTest, self).__init__(*args, **kwargs)
+
+	@classmethod
+	def tearDownClass(cls):
+		stop_all()
+
+	def test_retention_redundancy_1(self):
+		"""purge backups using redundancy-based retention policy"""
+		node = self.make_bnode('retention_redundancy_1',
+			base_dir="tmp_dirs/retention/retention_redundancy_1")
+		node.start()
+
+		self.init_pb(node)
+		with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
+			conf.write("REDUNDANCY=1\n")
+
+		# Make backups to be purged
+		self.backup_pb(node)
+		self.backup_pb(node, backup_type="page")
+		# Make backups to be keeped
+		self.backup_pb(node)
+		self.backup_pb(node, backup_type="page")
+
+		self.assertEqual(len(self.show_pb(node)), 4)
+
+		# Purge backups
+		self.retention_purge_pb(node)
+		self.assertEqual(len(self.show_pb(node)), 2)
+
+		node.stop()
+
+	def test_retention_window_2(self):
+		"""purge backups using window-based retention policy"""
+		node = self.make_bnode('retention_window_2',
+			base_dir="tmp_dirs/retention/retention_window_2")
+		node.start()
+
+		self.init_pb(node)
+		with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
+			conf.write("REDUNDANCY=1\n")
+			conf.write("WINDOW=2\n")
+
+		# All backups will be keeped
+		self.backup_pb(node)
+		self.backup_pb(node, backup_type="page")
+		self.backup_pb(node)
+		self.backup_pb(node, backup_type="page")		
+
+		self.assertEqual(len(self.show_pb(node)), 4)
+
+		# Purge backups
+		self.retention_purge_pb(node)
+		self.assertEqual(len(self.show_pb(node)), 4)
+
+		node.stop()

tests/validate_test.py

@@ -58,4 +58,4 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
 
 		id_backup = self.show_pb(node)[0].id
 		res = self.validate_pb(node, id_backup, options=['--xid=%s' % target_xid])
-		self.assertIn(six.b("incorrect resource manager data checksum in record"), res)
+		self.assertIn(six.b("could not read WAL record at"), res)

validate.c

@@ -30,7 +30,7 @@ void do_validate_last(void)
 	parray	*backup_list;
 	bool	another_pg_probackup = false;
 
-	ret = catalog_lock();
+	ret = catalog_lock(false);
 	if (ret == 1)
 		another_pg_probackup = true;
 
@@ -85,7 +85,7 @@ int do_validate(time_t backup_id,
 	pgBackup *base_backup = NULL;
 	bool backup_id_found = false;
 
-	catalog_lock();
+	catalog_lock(false);
 
 	rt = checkIfCreateRecoveryConf(target_time, target_xid, target_inclusive);
 	if (rt == NULL)