diff --git a/src/delete.c b/src/delete.c index e9461957..e114d44c 100644 --- a/src/delete.c +++ b/src/delete.c @@ -227,6 +227,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg uint32 actual_window = 0; uint64 actual_space = 0; + elog(INFO, "retention_redundancy %d, retention_size %ld", + instance_config.retention_redundancy, instance_config.retention_size); /* Calculate n_full_backups, total_backups_size and days_threshold */ if (instance_config.retention_redundancy > 0 || instance_config.retention_size > 0) @@ -241,8 +243,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg n_full_backups++; /* Consider only valid FULL backups for Redundancy */ - if (((instance_config.retention_redundancy > 0 ) && - backup->backup_mode == BACKUP_MODE_FULL && + if ((backup->backup_mode == BACKUP_MODE_FULL && (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE))) { @@ -255,10 +256,13 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg parray_append(redundancy_full_backup_list, backup); } + } - if ((instance_config.retention_size > 0) && - (total_backups_size <= instance_config.retention_size * 1024 * 1024)) + /* Consider only finished backups for retention_size */ + if ((backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) + && (instance_config.retention_size > 0) && + (total_backups_size <= instance_config.retention_size * 1024)) { if (!size_backup_list) size_backup_list = parray_new(); @@ -336,7 +340,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg (((instance_config.retention_redundancy == 0) || !redundancy_keep) && ((instance_config.retention_size == 0) || !size_keep))) { - elog(INFO, "Check if backup %s in needed by retention", base36enc(backup->start_time)); + elog(LOG, "Check if backup %s in needed by retention", base36enc(backup->start_time)); /* This backup is not guarded by retention * * Redundancy = 1 @@ -366,7 +370,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg } /* Add backup to purge_list */ - elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time)); + elog(INFO, "Mark backup %s for purge.", base36enc(backup->start_time)); parray_append(to_purge_list, backup); /* Calculate residuary space */ @@ -382,6 +386,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg && merge_expired) { pgBackup *full_backup = find_parent_full_backup(backup); + elog(LOG, "Find parent backup for %s to merge", base36enc(backup->start_time)); if (!parray_bsearch(to_purge_list, full_backup, pgBackupCompareIdDesc)) parray_append(to_purge_list, full_backup); if (!parray_bsearch(to_keep_list, backup, pgBackupCompareIdDesc)) @@ -444,7 +449,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg */ cur_full_backup_num = 1; - actual_space = 0; + //actual_space = 0; for (i = 0; i < parray_num(backup_list); i++) { char *action = "Active"; @@ -452,7 +457,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - actual_space += backup->data_bytes; + //actual_space += backup->data_bytes; if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc)) action = "Expired"; @@ -475,8 +480,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg instance_config.retention_redundancy, actual_window, pinning_window ? pinning_window : instance_config.retention_window, - (float)total_backups_size/(1024L * 1024L * 1024L), - (float)instance_config.retention_size, // /(1024L * 1024L), + (float)total_backups_size/(1024L * 1024L *1024L), + (float)instance_config.retention_size /(1024L *1024L), action); /* Only valid full backups are count to something */ diff --git a/tests/retention.py b/tests/retention.py index 11f305f9..c7f2fdf5 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -2596,21 +2596,24 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.show_pb(backup_dir, 'node')[0]['status'], 'OK') - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'], - 'RUNNING') +# self.assertTrue( +# self.show_pb(backup_dir, 'node')[1]['status'], +# 'RUNNING') - self.assertTrue( - self.show_pb(backup_dir, 'node')[2]['status'], - 'OK') +# self.assertTrue( +# self.show_pb(backup_dir, 'node')[2]['status'], +# 'OK') - self.assertEqual( - len(self.show_pb(backup_dir, 'node')), - 6) +# self.assertEqual( +# len(self.show_pb(backup_dir, 'node')), +# 6) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) self.del_test_dir(module_name, fname) - def test_retention_size(self): + def test_retention_size_save_all(self): """purge backups using size-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -2638,9 +2641,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Purge backups if their summ size more than 100 Mb + # Purge backups if their summ size more than 150 Mb (set value in Kb) self.delete_expired( - backup_dir, 'node', options=['--retention-size=200', '--expired']) + backup_dir, 'node', options=['--retention-size=153600', '--expired']) print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) @@ -2654,7 +2657,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) - def test_retention_size_2(self): + def test_retention_size_save_not_all(self): """purge backups using size-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -2679,9 +2682,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Purge backups if their summ size more than 50 Mb + # Purge backups if their summ size more than 50 Mb (set value in Kb) self.delete_expired( - backup_dir, 'node', options=['--retention-size=50', '--expired']) + backup_dir, 'node', options=['--retention-size=51200', '--expired']) print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) @@ -2692,7 +2695,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) - def test_retention_size_3(self): + def test_retention_size_delete_all(self): """purge backups using size-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -2705,37 +2708,29 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.slow_start() - self.set_config( - backup_dir, 'node', options=['--retention-size=250','--delete-expired']) - # take FULL BACKUP self.backup_node(backup_dir, 'node', node) - self.delete_expired( - backup_dir, 'node', options=['--retention-size=250', '--delete-expired']) - print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) # Take second FULL BACKUP self.backup_node(backup_dir, 'node', node) - self.delete_expired( - backup_dir, 'node', options=['--retention-size=250', '--delete-expired']) - print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) # Take third FULL BACKUP self.backup_node(backup_dir, 'node', node) + # Purge backups if their summ size more than 25 Mb self.delete_expired( - backup_dir, 'node', options=['--retention-size=25', '--delete-expired']) + backup_dir, 'node', options=['--retention-size=25600', '--delete-expired']) print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # We should get all 1 backups removed + # We should get all backups removed self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) # Clean after yourself @@ -2772,14 +2767,14 @@ class RetentionTest(ProbackupTest, unittest.TestCase): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Purge backups if their summ size more than 10 Mb + # Purge backups if their summ size more than 50 Mb self.delete_expired( - backup_dir, 'node', options=['--retention-size=10', '--merge-expired']) + backup_dir, 'node', options=['--retention-size=51200', '--merge-expired']) print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # We should get all 3 backups removed + # We should get 1 backup saved after merge self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) # Clean after yourself @@ -2799,7 +2794,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.slow_start() - # take FULL backup + # take FULL backup self.backup_node(backup_dir, 'node', node) # take PAGE backup @@ -2814,15 +2809,96 @@ class RetentionTest(ProbackupTest, unittest.TestCase): page_id_a3 = self.backup_node( backup_dir, 'node', node, backup_type='page') - # Purge backups if their summ size more than 10 Mb + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # Purge backups if their summ size more than 50 Mb self.delete_expired( - backup_dir, 'node', options=['--retention-size=10', '--expired']) + backup_dir, 'node', options=['--retention-size=51200', '--expired']) print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # We should get all 3 backups saved + # We should get all 4 backups saved self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_retention_size_redundancy_interleaved_1(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup + self.backup_node(backup_dir, 'node', node) + + # take FULL backup + self.backup_node(backup_dir, 'node', node) + + # take FULL backup + self.backup_node(backup_dir, 'node', node) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.delete_expired( + backup_dir, 'node', options=['--retention-size=10240', + '--retention-redundancy=2', '--expired']) + + # We should get only 1 backup saved despite of it's size more than retention_size + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_retention_size_redundancy_interleaved_2(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup + self.backup_node(backup_dir, 'node', node) + + # take FULL backup + self.backup_node(backup_dir, 'node', node) + + # take FULL backup + self.backup_node(backup_dir, 'node', node) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.delete_expired( + backup_dir, 'node', options=['--retention-size=1024000', + '--retention-redundancy=1', '--expired']) + + # We should get all 3 backup saved + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # Clean after yourself self.del_test_dir(module_name, fname)