db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Fixed problems with dimension of retention_size in tests. Fixed adding unfinished backups to list for saving and fail of test_concurrent_running_full_backup. Added tests for checking retention interleaved.

Browse Source
This commit is contained in:
d.lepikhova
2022-01-03 23:26:59 +05:00
parent 989dbf7d2a
commit eb6684f9ad
2 changed files with 125 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/delete.c
+15 -10
View File
@@ -227,6 +227,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
uint32 actual_window = 0;
uint64 actual_space = 0;
elog(INFO, "retention_redundancy %d, retention_size %ld",
instance_config.retention_redundancy, instance_config.retention_size);
/* Calculate n_full_backups, total_backups_size and days_threshold */
if (instance_config.retention_redundancy > 0 ||
instance_config.retention_size > 0)
@@ -241,8 +243,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
n_full_backups++;
/* Consider only valid FULL backups for Redundancy */
if (((instance_config.retention_redundancy > 0 ) &&
backup->backup_mode == BACKUP_MODE_FULL &&
if ((backup->backup_mode == BACKUP_MODE_FULL &&
(backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE)))
{
@@ -255,10 +256,13 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
parray_append(redundancy_full_backup_list, backup);
}
}
if ((instance_config.retention_size > 0) &&
(total_backups_size <= instance_config.retention_size * 1024 * 1024))
/* Consider only finished backups for retention_size */
if ((backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE)
&& (instance_config.retention_size > 0) &&
(total_backups_size <= instance_config.retention_size * 1024))
{
if (!size_backup_list)
size_backup_list = parray_new();
@@ -336,7 +340,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
(((instance_config.retention_redundancy == 0) || !redundancy_keep) &&
((instance_config.retention_size == 0) || !size_keep)))
{
elog(INFO, "Check if backup %s in needed by retention", base36enc(backup->start_time));
elog(LOG, "Check if backup %s in needed by retention", base36enc(backup->start_time));
/* This backup is not guarded by retention
*
* Redundancy = 1
@@ -366,7 +370,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
}
/* Add backup to purge_list */
elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time));
elog(INFO, "Mark backup %s for purge.", base36enc(backup->start_time));
parray_append(to_purge_list, backup);
/* Calculate residuary space */
@@ -382,6 +386,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
&& merge_expired)
{
pgBackup *full_backup = find_parent_full_backup(backup);
elog(LOG, "Find parent backup for %s to merge", base36enc(backup->start_time));
if (!parray_bsearch(to_purge_list, full_backup, pgBackupCompareIdDesc))
parray_append(to_purge_list, full_backup);
if (!parray_bsearch(to_keep_list, backup, pgBackupCompareIdDesc))
@@ -444,7 +449,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
*/
cur_full_backup_num = 1;
actual_space = 0;
//actual_space = 0;
for (i = 0; i < parray_num(backup_list); i++)
{
char *action = "Active";
@@ -452,7 +457,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
actual_space += backup->data_bytes;
//actual_space += backup->data_bytes;
if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc))
action = "Expired";
@@ -475,8 +480,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
instance_config.retention_redundancy,
actual_window,
pinning_window ? pinning_window : instance_config.retention_window,
(float)total_backups_size/(1024L * 1024L * 1024L),
(float)instance_config.retention_size, // /(1024L * 1024L),
(float)total_backups_size/(1024L * 1024L *1024L),
(float)instance_config.retention_size /(1024L *1024L),
action);
/* Only valid full backups are count to something */
tests/retention.py
+110 -34
View File
@@ -2596,21 +2596,24 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.show_pb(backup_dir, 'node')[0]['status'],
'OK')
self.assertTrue(
self.show_pb(backup_dir, 'node')[1]['status'],
'RUNNING')
# self.assertTrue(
# self.show_pb(backup_dir, 'node')[1]['status'],
# 'RUNNING')
self.assertTrue(
self.show_pb(backup_dir, 'node')[2]['status'],
'OK')
# self.assertTrue(
# self.show_pb(backup_dir, 'node')[2]['status'],
# 'OK')
self.assertEqual(
len(self.show_pb(backup_dir, 'node')),
6)
# self.assertEqual(
# len(self.show_pb(backup_dir, 'node')),
# 6)
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
self.del_test_dir(module_name, fname)
def test_retention_size(self):
def test_retention_size_save_all(self):
"""purge backups using size-based retention policy"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
@@ -2638,9 +2641,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# Purge backups if their summ size more than 100 Mb
# Purge backups if their summ size more than 150 Mb (set value in Kb)
self.delete_expired(
backup_dir, 'node', options=['--retention-size=200', '--expired'])
backup_dir, 'node', options=['--retention-size=153600', '--expired'])
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
@@ -2654,7 +2657,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_retention_size_2(self):
def test_retention_size_save_not_all(self):
"""purge backups using size-based retention policy"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
@@ -2679,9 +2682,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# Purge backups if their summ size more than 50 Mb
# Purge backups if their summ size more than 50 Mb (set value in Kb)
self.delete_expired(
backup_dir, 'node', options=['--retention-size=50', '--expired'])
backup_dir, 'node', options=['--retention-size=51200', '--expired'])
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
@@ -2692,7 +2695,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_retention_size_3(self):
def test_retention_size_delete_all(self):
"""purge backups using size-based retention policy"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
@@ -2705,37 +2708,29 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
self.set_config(
backup_dir, 'node', options=['--retention-size=250','--delete-expired'])
# take FULL BACKUP
self.backup_node(backup_dir, 'node', node)
self.delete_expired(
backup_dir, 'node', options=['--retention-size=250', '--delete-expired'])
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# Take second FULL BACKUP
self.backup_node(backup_dir, 'node', node)
self.delete_expired(
backup_dir, 'node', options=['--retention-size=250', '--delete-expired'])
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# Take third FULL BACKUP
self.backup_node(backup_dir, 'node', node)
# Purge backups if their summ size more than 25 Mb
self.delete_expired(
backup_dir, 'node', options=['--retention-size=25', '--delete-expired'])
backup_dir, 'node', options=['--retention-size=25600', '--delete-expired'])
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# We should get all 1 backups removed
# We should get all backups removed
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0)
# Clean after yourself
@@ -2772,14 +2767,14 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# Purge backups if their summ size more than 10 Mb
# Purge backups if their summ size more than 50 Mb
self.delete_expired(
backup_dir, 'node', options=['--retention-size=10', '--merge-expired'])
backup_dir, 'node', options=['--retention-size=51200', '--merge-expired'])
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# We should get all 3 backups removed
# We should get 1 backup saved after merge
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1)
# Clean after yourself
@@ -2799,7 +2794,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# take FULL backup
# take FULL backup
self.backup_node(backup_dir, 'node', node)
# take PAGE backup
@@ -2814,15 +2809,96 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
page_id_a3 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Purge backups if their summ size more than 10 Mb
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# Purge backups if their summ size more than 50 Mb
self.delete_expired(
backup_dir, 'node', options=['--retention-size=10', '--expired'])
backup_dir, 'node', options=['--retention-size=51200', '--expired'])
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# We should get all 3 backups saved
# We should get all 4 backups saved
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4)
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_retention_size_redundancy_interleaved_1(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# take FULL backup
self.backup_node(backup_dir, 'node', node)
# take FULL backup
self.backup_node(backup_dir, 'node', node)
# take FULL backup
self.backup_node(backup_dir, 'node', node)
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
self.delete_expired(
backup_dir, 'node', options=['--retention-size=10240',
'--retention-redundancy=2', '--expired'])
# We should get only 1 backup saved despite of it's size more than retention_size
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2)
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_retention_size_redundancy_interleaved_2(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# take FULL backup
self.backup_node(backup_dir, 'node', node)
# take FULL backup
self.backup_node(backup_dir, 'node', node)
# take FULL backup
self.backup_node(backup_dir, 'node', node)
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
self.delete_expired(
backup_dir, 'node', options=['--retention-size=1024000',
'--retention-redundancy=1', '--expired'])
# We should get all 3 backup saved
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3)
print(self.show_pb(
backup_dir, 'node', as_json=False, as_text=True))
# Clean after yourself
self.del_test_dir(module_name, fname)