Merge branch 'issue_120'

doc/pgprobackup.xml

@@ -3257,6 +3257,17 @@ pg_probackup delete -B <replaceable>backup_dir</replaceable> --instance <replace
       all the available backups according to the current retention
       policy, without performing any irreversible actions.
     </para>
+    <para>
+      To delete all backups with specific status, use the <option>--status</option>:
+    </para>
+    <programlisting>
+pg_probackup delete -B <replaceable>backup_dir</replaceable> --instance <replaceable>instance_name</replaceable> --status=ERROR
+    </programlisting>
+
+    <para>
+      Deleting backups by status ignores established retention policies.
+    </para>
+
 </refsect2>
 </refsect1>
 
@@ -3935,10 +3946,9 @@ pg_probackup merge -B <replaceable>backup_dir</replaceable> --instance <replacea
       <programlisting>
 pg_probackup delete -B <replaceable>backup_dir</replaceable> --instance <replaceable>instance_name</replaceable>
 [--help] [-j <replaceable>num_threads</replaceable>] [--progress]
-[--retention-redundancy=<replaceable>redundancy</replaceable>][--retention-window=<replaceable>window</replaceable>][--wal-depth=<replaceable>wal_depth</replaceable>]
-[--delete-wal] {-i <replaceable>backup_id</replaceable> | --delete-expired [--merge-expired] | --merge-expired}
-[--dry-run]
-[<replaceable>logging_options</replaceable>]
+[--retention-redundancy=<replaceable>redundancy</replaceable>][--retention-window=<replaceable>window</replaceable>][--wal-depth=<replaceable>wal_depth</replaceable>] [--delete-wal]
+{-i <replaceable>backup_id</replaceable> | --delete-expired [--merge-expired] | --merge-expired | --status=backup_status}
+[--dry-run] [<replaceable>logging_options</replaceable>]
 </programlisting>
       <para>
         Deletes backup with specified <replaceable>backup_id</replaceable>

src/catalog.c

@@ -2447,3 +2447,23 @@ get_backup_index_number(parray *backup_list, pgBackup *backup)
 	elog(WARNING, "Failed to find backup %s", base36enc(backup->start_time));
 	return -1;
 }
+
+/* On backup_list lookup children of target_backup and append them to append_list */
+void
+append_children(parray *backup_list, pgBackup *target_backup, parray *append_list)
+{
+	int i;
+
+	for (i = 0; i < parray_num(backup_list); i++)
+	{
+		pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
+
+		/* check if backup is descendant of target backup */
+		if (is_parent(target_backup->start_time, backup, false))
+		{
+			/* if backup is already in the list, then skip it */
+			if (!parray_contains(append_list, backup))
+				parray_append(append_list, backup);
+		}
+	}
+}

src/delete.c

@@ -123,7 +123,7 @@ do_delete(time_t backup_id)
  * which FULL backup should be keeped for redundancy obligation(only valid do),
  * but if invalid backup is not guarded by retention - it is removed
  */
-int do_retention(void)
+void do_retention(void)
 {
 	parray	   *backup_list = NULL;
 	parray	   *to_keep_list = parray_new();
@@ -154,7 +154,7 @@ int do_retention(void)
 			/* Retention is disabled but we still can cleanup wal */
 			elog(WARNING, "Retention policy is not set");
 			if (!delete_wal)
-				return 0;
+				return;
 		}
 		else
 			/* At least one retention policy is active */
@@ -196,9 +196,6 @@ int do_retention(void)
 	parray_free(backup_list);
 	parray_free(to_keep_list);
 	parray_free(to_purge_list);
-
-	return 0;
-
 }
 
 /* Evaluate every backup by retention policies and populate purge and keep lists.
@@ -1023,3 +1020,107 @@ do_delete_instance(void)
 	elog(INFO, "Instance '%s' successfully deleted", instance_name);
 	return 0;
 }
+
+/* Delete all backups of given status in instance */
+void
+do_delete_status(InstanceConfig *instance_config, const char *status)
+{
+	int         i;
+	parray     *backup_list, *delete_list;
+	const char *pretty_status;
+	int         n_deleted = 0, n_found = 0;
+	size_t      size_to_delete = 0;
+	char        size_to_delete_pretty[20];
+	pgBackup   *backup;
+
+	BackupStatus status_for_delete = str2status(status);
+	delete_list = parray_new();
+
+	if (status_for_delete == BACKUP_STATUS_INVALID)
+		elog(ERROR, "Unknown value for '--status' option: '%s'", status);
+
+	/*
+	 * User may have provided status string in lower case, but
+	 * we should print backup statuses consistently with show command,
+	 * so convert it.
+	 */
+	pretty_status = status2str(status_for_delete);
+
+	backup_list = catalog_get_backup_list(instance_config->name, INVALID_BACKUP_ID);
+
+	if (parray_num(backup_list) == 0)
+	{
+		elog(WARNING, "Instance '%s' has no backups", instance_config->name);
+		return;
+	}
+
+	if (dry_run)
+		elog(INFO, "Deleting all backups with status '%s' in dry run mode", pretty_status);
+	else
+		elog(INFO, "Deleting all backups with status '%s'", pretty_status);
+
+	/* Selects backups with specified status and their children into delete_list array. */
+	for (i = 0; i < parray_num(backup_list); i++)
+	{
+		backup = (pgBackup *) parray_get(backup_list, i);
+
+		if (backup->status == status_for_delete)
+		{
+			n_found++;
+
+			/* incremental backup can be already in delete_list due to append_children() */
+			if (parray_contains(delete_list, backup))
+				continue;
+			parray_append(delete_list, backup);
+
+			append_children(backup_list, backup, delete_list);
+		}
+	}
+
+	parray_qsort(delete_list, pgBackupCompareIdDesc);
+
+	/* delete and calculate free size from delete_list */
+	for (i = 0; i < parray_num(delete_list); i++)
+	{
+		backup = (pgBackup *)parray_get(delete_list, i);
+
+		elog(INFO, "Backup %s with status %s %s be deleted",
+			base36enc(backup->start_time), status2str(backup->status), dry_run ? "can" : "will");
+
+		size_to_delete += backup->data_bytes;
+		if (backup->stream)
+			size_to_delete += backup->wal_bytes;
+
+		if (!dry_run && lock_backup(backup))
+			delete_backup_files(backup);
+
+		n_deleted++;
+	}
+
+	/* Inform about data size to free */
+	if (size_to_delete >= 0)
+	{
+		pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty));
+		elog(INFO, "Resident data size to free by delete of %i backups: %s",
+			n_deleted, size_to_delete_pretty);
+	}
+
+	/* delete selected backups */
+	if (!dry_run && n_deleted > 0)
+		elog(INFO, "Successfully deleted %i %s from instance '%s'",
+			n_deleted, n_deleted == 1 ? "backup" : "backups",
+			instance_config->name);
+
+
+	if (n_found == 0)
+		elog(WARNING, "Instance '%s' has no backups with status '%s'",
+			instance_config->name, pretty_status);
+
+	// we don`t do WAL purge here, because it is impossible to correctly handle
+	// dry-run case.
+
+	/* Cleanup */
+	parray_free(delete_list);
+	parray_walk(backup_list, pgBackupFree);
+	parray_free(backup_list);
+}

src/help.c

@@ -193,7 +193,8 @@ help_pg_probackup(void)
 	printf(_("                 [--retention-redundancy=retention-redundancy]\n"));
 	printf(_("                 [--retention-window=retention-window]\n"));
 	printf(_("                 [--wal-depth=wal-depth]\n"));
-	printf(_("                 [--delete-wal] [-i backup-id | --delete-expired | --merge-expired]\n"));
+	printf(_("                 [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n"));
+	printf(_("                 [--delete-wal]\n"));
 	printf(_("                 [--dry-run]\n"));
 	printf(_("                 [--help]\n"));
 
@@ -638,6 +639,7 @@ help_delete(void)
 	printf(_("      --wal-depth=wal-depth        number of latest valid backups per timeline that must\n"));
 	printf(_("                                   retain the ability to perform PITR; 0 disables; (default: 0)\n"));
 	printf(_("      --dry-run                    perform a trial run without any changes\n"));
+	printf(_("      --status=backup_status       delete all backups with specified status\n"));
 
 	printf(_("\n  Logging options:\n"));
 	printf(_("      --log-level-console=log-level-console\n"));

src/pg_probackup.c

@@ -117,7 +117,7 @@ bool		delete_expired = false;
 bool		merge_expired = false;
 bool		force = false;
 bool		dry_run = false;
-
+static char *delete_status = NULL;
 /* compression options */
 bool 		compress_shortcut = false;
 
@@ -209,6 +209,8 @@ static ConfigOption cmd_options[] =
 	/* delete options */
 	{ 'b', 145, "wal",				&delete_wal,		SOURCE_CMD_STRICT },
 	{ 'b', 146, "expired",			&delete_expired,	SOURCE_CMD_STRICT },
+	{ 's', 172, "status",			&delete_status,		SOURCE_CMD_STRICT },
+
 	/* TODO not implemented yet */
 	{ 'b', 147, "force",			&force,				SOURCE_CMD_STRICT },
 	/* compression options */
@@ -821,13 +823,20 @@ main(int argc, char *argv[])
 				elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together");
 			if (merge_expired && backup_id_string)
 				elog(ERROR, "You cannot specify --merge-expired and (-i, --backup-id) options together");
-			if (!delete_expired && !merge_expired && !delete_wal && !backup_id_string)
+			if (delete_status && backup_id_string)
+				elog(ERROR, "You cannot specify --status and (-i, --backup-id) options together");
+			if (!delete_expired && !merge_expired && !delete_wal && delete_status == NULL && !backup_id_string)
 				elog(ERROR, "You must specify at least one of the delete options: "
-								"--delete-expired |--delete-wal |--merge-expired |(-i, --backup-id)");
+								"--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)");
 			if (!backup_id_string)
-				return do_retention();
+			{
+				if (delete_status)
+					do_delete_status(&instance_config, delete_status);
+				else
+					do_retention();
+			}
 			else
-				do_delete(current.backup_id);
+					do_delete(current.backup_id);
 			break;
 		case MERGE_CMD:
 			do_merge(current.backup_id);

src/pg_probackup.h

@@ -722,8 +722,9 @@ extern int do_show(const char *instance_name, time_t requested_backup_id, bool s
 /* in delete.c */
 extern void do_delete(time_t backup_id);
 extern void delete_backup_files(pgBackup *backup);
-extern int do_retention(void);
+extern void do_retention(void);
 extern int do_delete_instance(void);
+extern void do_delete_status(InstanceConfig *instance_config, const char *status);
 
 /* in fetch.c */
 extern char *slurpFile(const char *datadir,
@@ -807,8 +808,8 @@ extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup)
 
 extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive);
 extern bool is_prolific(parray *backup_list, pgBackup *target_backup);
-extern bool in_backup_list(parray *backup_list, pgBackup *target_backup);
 extern int get_backup_index_number(parray *backup_list, pgBackup *backup);
+extern void append_children(parray *backup_list, pgBackup *target_backup, parray *append_list);
 extern bool launch_agent(void);
 extern void launch_ssh(char* argv[]);
 extern void wait_ssh(void);
@@ -953,6 +954,7 @@ extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_loc
 
 extern void time2iso(char *buf, size_t len, time_t time);
 extern const char *status2str(BackupStatus status);
+extern BackupStatus str2status(const char *status);
 extern const char *base36enc(long unsigned int value);
 extern char *base36enc_dup(long unsigned int value);
 extern long unsigned int base36dec(const char *text);

src/util.c

@@ -18,6 +18,21 @@
 
 #include <sys/stat.h>
 
+static const char *statusName[] =
+{
+	"UNKNOWN",
+	"OK",
+	"ERROR",
+	"RUNNING",
+	"MERGING",
+	"MERGED",
+	"DELETING",
+	"DELETED",
+	"DONE",
+	"ORPHAN",
+	"CORRUPT"
+};
+
 const char *
 base36enc(long unsigned int value)
 {
@@ -462,22 +477,21 @@ parse_program_version(const char *program_version)
 const char *
 status2str(BackupStatus status)
 {
-	static const char *statusName[] =
-	{
-		"UNKNOWN",
-		"OK",
-		"ERROR",
-		"RUNNING",
-		"MERGING",
-		"MERGED",
-		"DELETING",
-		"DELETED",
-		"DONE",
-		"ORPHAN",
-		"CORRUPT"
-	};
 	if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status)
 		return "UNKNOWN";
 
 	return statusName[status];
 }
+
+BackupStatus
+str2status(const char *status)
+{
+	BackupStatus i;
+
+	for (i = BACKUP_STATUS_INVALID; i <= BACKUP_STATUS_CORRUPT; i++)
+	{
+		if (pg_strcasecmp(status, statusName[i]) == 0) return i;
+	}
+
+	return BACKUP_STATUS_INVALID;
+}

src/utils/parray.c

@@ -197,3 +197,16 @@ parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const
 {
 	return bsearch(&key, array->data, array->used, sizeof(void *), compare);
 }
+
+/* checks that parray contains element */
+bool parray_contains(parray *array, void *elem)
+{
+	int i;
+
+	for (i = 0; i < parray_num(array); i++)
+	{
+		if (parray_get(array, i) == elem)
+			return true;
+	}
+	return false;
+}

src/utils/parray.h

@@ -30,6 +30,7 @@ extern size_t parray_num(const parray *array);
 extern void parray_qsort(parray *array, int(*compare)(const void *, const void *));
 extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *));
 extern void parray_walk(parray *array, void (*action)(void *));
+extern bool parray_contains(parray *array, void *elem);
 
 #endif /* PARRAY_H */
 

tests/delete.py

@@ -801,3 +801,75 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    def test_delete_error_backups(self):
+        """delete increment and all after him"""
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            initdb_params=['--data-checksums'])
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        # full backup mode
+        self.backup_node(backup_dir, 'node', node)
+        # page backup mode
+        self.backup_node(backup_dir, 'node', node, backup_type="page")
+
+        # Take FULL BACKUP
+        backup_id_a = self.backup_node(backup_dir, 'node', node)
+        # Take PAGE BACKUP
+        backup_id_b = self.backup_node(backup_dir, 'node', node, backup_type="page")
+
+        backup_id_c = self.backup_node(backup_dir, 'node', node, backup_type="page")
+
+        backup_id_d = self.backup_node(backup_dir, 'node', node, backup_type="page")
+
+        # full backup mode
+        self.backup_node(backup_dir, 'node', node)
+        self.backup_node(backup_dir, 'node', node, backup_type="page")
+        backup_id_e = self.backup_node(backup_dir, 'node', node, backup_type="page")
+        self.backup_node(backup_dir, 'node', node, backup_type="page")
+
+        # Change status to ERROR
+        self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
+        self.change_backup_status(backup_dir, 'node', backup_id_c, 'ERROR')
+        self.change_backup_status(backup_dir, 'node', backup_id_e, 'ERROR')
+
+        print(self.show_pb(backup_dir, as_text=True, as_json=False))
+
+        show_backups = self.show_pb(backup_dir, 'node')
+        self.assertEqual(len(show_backups), 10)
+
+        # delete error backups
+        output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR', '--dry-run'])
+        show_backups = self.show_pb(backup_dir, 'node')
+        self.assertEqual(len(show_backups), 10)
+
+        self.assertIn(
+            "Deleting all backups with status 'ERROR' in dry run mode",
+            output)
+
+        self.assertIn(
+            "INFO: Backup {0} with status OK can be deleted".format(backup_id_d),
+            output)
+
+        print(self.show_pb(backup_dir, as_text=True, as_json=False))
+
+        show_backups = self.show_pb(backup_dir, 'node')
+        output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR'])
+        print(output)
+        show_backups = self.show_pb(backup_dir, 'node')
+        self.assertEqual(len(show_backups), 4)
+
+        self.assertEqual(show_backups[0]['status'], "OK")
+        self.assertEqual(show_backups[1]['status'], "OK")
+        self.assertEqual(show_backups[2]['status'], "OK")
+        self.assertEqual(show_backups[3]['status'], "OK")
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)