Fixes for retention and validate commands

catalog.c

@@ -314,7 +314,7 @@ pgBackupWriteResultSection(FILE *out, pgBackup *backup)
 	}
 }
 
-/* create backup.ini */
+/* create backup.conf */
 void
 pgBackupWriteIni(pgBackup *backup)
 {
@@ -324,7 +324,7 @@ pgBackupWriteIni(pgBackup *backup)
 	pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_CONF_FILE);
 	fp = fopen(ini_path, "wt");
 	if (fp == NULL)
-		elog(ERROR, "cannot open INI file \"%s\": %s", ini_path,
+		elog(ERROR, "cannot open configuration file \"%s\": %s", ini_path,
 			strerror(errno));
 
 	/* configuration section */

delete.c

@@ -158,6 +158,7 @@ do_retention_purge(void)
 	XLogRecPtr	oldest_lsn = InvalidXLogRecPtr;
 	TimeLineID	oldest_tli;
 	int			ret;
+	bool		keep_next_backup = true; /* Do not delete first full backup */
 
 	if (retention_redundancy > 0)
 		elog(LOG, "REDUNDANCY=%u", retention_redundancy);
@@ -190,6 +191,10 @@ do_retention_purge(void)
 		pgBackup   *backup = (pgBackup *) parray_get(backup_list, i);
 		uint32		backup_num_evaluate = backup_num;
 
+		/* Consider only validated and correct backups */
+		if (backup->status != BACKUP_STATUS_OK)
+			continue;
+
 		/*
 		 * When a validate full backup was found, we can delete the
 		 * backup that is older than it using the number of generations.
@@ -198,12 +203,24 @@ do_retention_purge(void)
 			backup_num++;
 
 		/* Evaluate if this backup is eligible for removal */
-		if (backup_num_evaluate + 1 <= retention_redundancy ||
-			(retention_window > 0 && backup->start_time >= days_threshold))
+		if (keep_next_backup ||
+			backup_num_evaluate + 1 <= retention_redundancy ||
+			(retention_window > 0 && backup->recovery_time >= days_threshold))
 		{
 			/* Save LSN and Timeline to remove unnecessary WAL segments */
 			oldest_lsn = backup->start_lsn;
 			oldest_tli = backup->tli;
+
+			/* Save parent backup of this incremental backup */
+			if (backup->backup_mode != BACKUP_MODE_FULL)
+				keep_next_backup = true;
+			/*
+			 * Previous incremental backup was kept or this is first backup
+			 * so do not delete this backup.
+			 */
+			else
+				keep_next_backup = false;
+
 			continue;
 		}
 

parsexlog.c

@@ -104,18 +104,20 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
 void
 validate_wal(pgBackup *backup,
 			 const char *archivedir,
-			 XLogRecPtr startpoint,
 			 time_t target_time,
-			 TransactionId recovery_target_xid,
+			 TransactionId target_xid,
 			 TimeLineID tli)
 {
+	XLogRecPtr	startpoint = backup->start_lsn;
 	XLogRecord *record;
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 	XLogPageReadPrivate private;
 	TransactionId last_xid = InvalidTransactionId;
 	TimestampTz last_time = 0;
-	char	timestamp[100];
+	char		timestamp[100];
+	bool		all_wal = false,
+				got_endpoint = false;
 
 	private.archivedir = archivedir;
 	private.tli = tli;
@@ -130,32 +132,39 @@ validate_wal(pgBackup *backup,
 		record = XLogReadRecord(xlogreader, startpoint, &errormsg);
 		if (record == NULL)
 		{
-			XLogRecPtr	errptr;
-
-			errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
-			if (recovery_target_xid == InvalidTransactionId && target_time == 0)
-			{
-				break;
-			}
-
 			if (errormsg)
-				elog(ERROR, "could not read WAL record at %X/%X: %s",
-						 (uint32) (errptr >> 32), (uint32) (errptr),
-						 errormsg);
-			else
-				elog(ERROR, "could not read WAL record at %X/%X",
-						 (uint32) (errptr >> 32),
-						 (uint32) (errptr));
+				elog(WARNING, "%s", errormsg);
+
+			break;
 		}
 
+		/* Got WAL record at stop_lsn */
+		if (xlogreader->ReadRecPtr == backup->stop_lsn)
+			got_endpoint = true;
+
 		timestamp_record = getRecordTimestamp(xlogreader, &last_time);
 		if (XLogRecGetXid(xlogreader) != InvalidTransactionId)
 			last_xid = XLogRecGetXid(xlogreader);
-		if (recovery_target_xid != InvalidTransactionId && recovery_target_xid == last_xid)
-			break;
 
-		if (target_time != 0 && timestamp_record && timestamptz_to_time_t(last_time) >= target_time)
+		/* Check target xid */
+		if (TransactionIdIsValid(target_xid) && target_xid == last_xid)
+		{
+			all_wal = true;
 			break;
+		}
+		/* Check target time */
+		else if (target_time != 0 && timestamp_record && timestamptz_to_time_t(last_time) >= target_time)
+		{
+			all_wal = true;
+			break;
+		}
+		/* Stop if there are no target xid and target time */
+		else if (!TransactionIdIsValid(target_xid) && target_time == 0 &&
+			xlogreader->ReadRecPtr == backup->stop_lsn)
+		{
+			all_wal = true;
+			break;
+		}
 
 		startpoint = InvalidXLogRecPtr; /* continue reading at next record */
 	}
@@ -167,7 +176,39 @@ validate_wal(pgBackup *backup,
 	if (last_xid == InvalidTransactionId)
 		last_xid = backup->recovery_xid;
 
-	elog(INFO, "Backup validation stopped on %s time and xid:" XID_FMT, timestamp, last_xid);
+	/* There are all need WAL records */
+	if (all_wal)
+		elog(INFO, "Backup validation stopped on %s time and xid:" XID_FMT,
+			 timestamp, last_xid);
+	/* There are not need WAL records */
+	else
+	{
+		if (!got_endpoint)
+			elog(ERROR, "there are not enough WAL records to restore from %X/%X to %X/%X",
+				 (uint32) (backup->start_lsn >> 32),
+				 (uint32) (backup->start_lsn),
+				 (uint32) (backup->stop_lsn >> 32),
+				 (uint32) (backup->stop_lsn));
+		else
+		{
+			if (target_time > 0)
+				time2iso(timestamp, lengthof(timestamp),
+						 timestamptz_to_time_t(target_time));
+
+			if (TransactionIdIsValid(target_xid) && target_time != 0)
+				elog(WARNING, "there are not WAL records to time %s and xid " XID_FMT,
+					 timestamp, target_xid);
+			else if (TransactionIdIsValid(target_xid))
+				elog(WARNING, "there are not WAL records to xid " XID_FMT,
+					 target_xid);
+			else if (target_time != 0)
+				elog(WARNING, "there are not WAL records to time %s ",
+					 timestamp);
+
+			elog(WARNING, "recovery can be done to time %s and xid " XID_FMT,
+				 timestamp, last_xid);
+		}
+	}
 
 	/* clean */
 	XLogReaderFree(xlogreader);

pg_probackup.h

@@ -356,9 +356,8 @@ extern void extractPageMap(const char *datadir,
 						   XLogRecPtr endpoint);
 extern void validate_wal(pgBackup *backup,
 						 const char *archivedir,
-						 XLogRecPtr startpoint,
 						 time_t target_time,
-						 TransactionId recovery_target_xid,
+						 TransactionId target_xid,
 						 TimeLineID tli);
 
 /* in util.c */

tests/validate_test.py

@@ -58,4 +58,4 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
 
 		id_backup = self.show_pb(node)[0].id
 		res = self.validate_pb(node, id_backup, options=['--xid=%s' % target_xid])
-		self.assertIn(six.b("could not read WAL record at"), res)
+		self.assertIn(six.b("there are not WAL records to xid"), res)

validate.c

@@ -74,16 +74,16 @@ int do_validate(time_t backup_id,
 				const char *target_inclusive,
 				TimeLineID target_tli)
 {
-	int		i;
-	int base_index;				/* index of base (full) backup */
-	int last_restored_index;	/* index of last restored database backup */
+	int			i;
+	int			base_index;				/* index of base (full) backup */
+	int			last_restored_index;	/* index of last restored database backup */
 	TimeLineID	backup_tli;
 	TimeLineID	newest_tli;
-	parray *timelines;
-	parray *backups;
+	parray	   *timelines;
+	parray	   *backups;
 	pgRecoveryTarget *rt = NULL;
-	pgBackup *base_backup = NULL;
-	bool backup_id_found = false;
+	pgBackup   *base_backup = NULL;
+	bool		backup_id_found = false;
 
 	catalog_lock(false);
 
@@ -187,15 +187,12 @@ base_backup_found:
 	}
 
 	/* and now we must check WALs */
-	{
-		pgBackup *backup = (pgBackup *) parray_get(backups, last_restored_index);
-		validate_wal(backup,
+	if (!stream_wal)
+		validate_wal((pgBackup *) parray_get(backups, last_restored_index),
 					 arclog_path,
-					 backup->start_lsn,
 					 rt->recovery_target_time,
 					 rt->recovery_target_xid,
 					 target_tli);
-	}
 
 	/* release catalog lock */
 	catalog_unlock();