1
0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-25 09:01:48 +02:00

Merge branch 'master' into issue_150

This commit is contained in:
Grigory Smolkin 2019-12-27 00:56:00 +03:00
commit fc81de2398
7 changed files with 165 additions and 94 deletions

View File

@ -2438,14 +2438,14 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> [--instance <replacea
</para>
<programlisting>
ARCHIVE INSTANCE 'node'
===================================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
===================================================================================================================
5 1 0/B000000 000000000000000B 000000000000000C 2 685kB 48.00 0 OK
4 3 0/18000000 0000000000000018 000000000000001A 3 648kB 77.00 0 OK
3 2 0/15000000 0000000000000015 0000000000000017 3 648kB 77.00 0 OK
2 1 0/B000108 000000000000000B 0000000000000015 5 892kB 94.00 1 DEGRADED
1 0 0/0 0000000000000001 000000000000000A 10 8774kB 19.00 1 OK
===================================================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
===================================================================================================================================
5 1 0/B000000 00000005000000000000000B 00000005000000000000000C 2 685kB 48.00 0 OK
4 3 0/18000000 000000040000000000000018 00000004000000000000001A 3 648kB 77.00 0 OK
3 2 0/15000000 000000030000000000000015 000000030000000000000017 3 648kB 77.00 0 OK
2 1 0/B000108 00000002000000000000000B 000000020000000000000015 5 892kB 94.00 1 DEGRADED
1 0 0/0 000000010000000000000001 00000001000000000000000A 10 8774kB 19.00 1 OK
</programlisting>
<para>
For each timeline, the following information is provided:
@ -2543,8 +2543,8 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> [--instance <replacea
"tli": 5,
"parent-tli": 1,
"switchpoint": "0/B000000",
"min-segno": "000000000000000B",
"max-segno": "000000000000000C",
"min-segno": "00000005000000000000000B",
"max-segno": "00000005000000000000000C",
"n-segments": 2,
"size": 685320,
"zratio": 48.00,
@ -2557,8 +2557,8 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> [--instance <replacea
"tli": 4,
"parent-tli": 3,
"switchpoint": "0/18000000",
"min-segno": "0000000000000018",
"max-segno": "000000000000001A",
"min-segno": "000000040000000000000018",
"max-segno": "00000004000000000000001A",
"n-segments": 3,
"size": 648625,
"zratio": 77.00,
@ -2571,8 +2571,8 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> [--instance <replacea
"tli": 3,
"parent-tli": 2,
"switchpoint": "0/15000000",
"min-segno": "0000000000000015",
"max-segno": "0000000000000017",
"min-segno": "000000030000000000000015",
"max-segno": "000000030000000000000017",
"n-segments": 3,
"size": 648911,
"zratio": 77.00,
@ -2585,8 +2585,8 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> [--instance <replacea
"tli": 2,
"parent-tli": 1,
"switchpoint": "0/B000108",
"min-segno": "000000000000000B",
"max-segno": "0000000000000015",
"min-segno": "00000002000000000000000B",
"max-segno": "000000020000000000000015",
"n-segments": 5,
"size": 892173,
"zratio": 94.00,
@ -2594,12 +2594,12 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> [--instance <replacea
"status": "DEGRADED",
"lost-segments": [
{
"begin-segno": "000000000000000D",
"end-segno": "000000000000000E"
"begin-segno": "00000002000000000000000D",
"end-segno": "00000002000000000000000E"
},
{
"begin-segno": "0000000000000010",
"end-segno": "0000000000000012"
"begin-segno": "000000020000000000000010",
"end-segno": "000000020000000000000012"
}
],
"backups": [
@ -2634,8 +2634,8 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> [--instance <replacea
"tli": 1,
"parent-tli": 0,
"switchpoint": "0/0",
"min-segno": "0000000000000001",
"max-segno": "000000000000000A",
"min-segno": "000000010000000000000001",
"max-segno": "00000001000000000000000A",
"n-segments": 10,
"size": 8774805,
"zratio": 19.00,
@ -2679,8 +2679,8 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> [--instance <replacea
"tli": 1,
"parent-tli": 0,
"switchpoint": "0/0",
"min-segno": "0000000000000001",
"max-segno": "000000000000000B",
"min-segno": "000000010000000000000001",
"max-segno": "00000001000000000000000B",
"n-segments": 11,
"size": 8860892,
"zratio": 20.00,
@ -3049,10 +3049,10 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> --instance node --arc
</programlisting>
<programlisting>
ARCHIVE INSTANCE 'node'
===============================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
===============================================================================================================
1 0 0/0 0000000000000001 0000000000000047 71 36MB 31.00 6 OK
===============================================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
===============================================================================================================================
1 0 0/0 000000010000000000000001 000000010000000000000047 71 36MB 31.00 6 OK
</programlisting>
<para>
WAL purge without <option>--wal-depth</option> cannot
@ -3063,10 +3063,10 @@ pg_probackup delete -B <replaceable>backup_dir</replaceable> --instance node --d
</programlisting>
<programlisting>
ARCHIVE INSTANCE 'node'
===============================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
===============================================================================================================
1 0 0/0 0000000000000002 0000000000000047 70 34MB 32.00 6 OK
===============================================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
===============================================================================================================================
1 0 0/0 000000010000000000000002 000000010000000000000047 70 34MB 32.00 6 OK
</programlisting>
<para>
If you would like, for example, to keep only those WAL
@ -3078,10 +3078,10 @@ pg_probackup delete -B <replaceable>backup_dir</replaceable> --instance node --d
</programlisting>
<programlisting>
ARCHIVE INSTANCE 'node'
================================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
================================================================================================================
1 0 0/0 0000000000000046 0000000000000047 2 143kB 228.00 6 OK
================================================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
================================================================================================================================
1 0 0/0 000000010000000000000046 000000010000000000000047 2 143kB 228.00 6 OK
</programlisting>
<para>
Alternatively, you can use the <option>--wal-depth</option>
@ -3092,10 +3092,10 @@ pg_probackup backup -B <replaceable>backup_dir</replaceable> --instance node -b
</programlisting>
<programlisting>
ARCHIVE INSTANCE 'node'
===============================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
===============================================================================================================
1 0 0/0 0000000000000048 0000000000000049 1 72kB 228.00 7 OK
===============================================================================================================================
TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status
===============================================================================================================================
1 0 0/0 000000010000000000000048 000000010000000000000049 1 72kB 228.00 7 OK
</programlisting>
<note>
<para>

View File

@ -697,8 +697,8 @@ catalog_get_timelines(InstanceConfig *instance)
char arclog_path[MAXPGPATH];
/* for fancy reporting */
char begin_segno_str[20];
char end_segno_str[20];
char begin_segno_str[XLOG_FNAME_LEN];
char end_segno_str[XLOG_FNAME_LEN];
/* read all xlog files that belong to this archive */
sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
@ -1132,8 +1132,8 @@ catalog_get_timelines(InstanceConfig *instance)
* covered by other larger interval.
*/
GetXLogSegName(begin_segno_str, interval->begin_segno, instance->xlog_seg_size);
GetXLogSegName(end_segno_str, interval->end_segno, instance->xlog_seg_size);
GetXLogFileName(begin_segno_str, tlinfo->tli, interval->begin_segno, instance->xlog_seg_size);
GetXLogFileName(end_segno_str, tlinfo->tli, interval->end_segno, instance->xlog_seg_size);
elog(LOG, "Timeline %i to stay reachable from timeline %i "
"protect from purge WAL interval between "
@ -1187,8 +1187,8 @@ catalog_get_timelines(InstanceConfig *instance)
else
interval->end_segno = segno;
GetXLogSegName(begin_segno_str, interval->begin_segno, instance->xlog_seg_size);
GetXLogSegName(end_segno_str, interval->end_segno, instance->xlog_seg_size);
GetXLogFileName(begin_segno_str, tlinfo->tli, interval->begin_segno, instance->xlog_seg_size);
GetXLogFileName(end_segno_str, tlinfo->tli, interval->end_segno, instance->xlog_seg_size);
elog(LOG, "Archive backup %s to stay consistent "
"protect from purge WAL interval "

View File

@ -806,8 +806,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo,
{
XLogSegNo FirstToDeleteSegNo;
XLogSegNo OldestToKeepSegNo = 0;
char first_to_del_str[20];
char oldest_to_keep_str[20];
char first_to_del_str[XLOG_FNAME_LEN];
char oldest_to_keep_str[XLOG_FNAME_LEN];
int rc;
int i;
size_t wal_size_logical = 0;
@ -842,8 +842,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo,
if (OldestToKeepSegNo > 0 && OldestToKeepSegNo > FirstToDeleteSegNo)
{
/* translate segno number into human readable format */
GetXLogSegName(first_to_del_str, FirstToDeleteSegNo, xlog_seg_size);
GetXLogSegName(oldest_to_keep_str, OldestToKeepSegNo, xlog_seg_size);
GetXLogFileName(first_to_del_str, tlinfo->tli, FirstToDeleteSegNo, xlog_seg_size);
GetXLogFileName(oldest_to_keep_str, tlinfo->tli, OldestToKeepSegNo, xlog_seg_size);
elog(INFO, "On timeline %i WAL segments between %s and %s %s be removed",
tlinfo->tli, first_to_del_str,
@ -870,8 +870,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo,
if (FirstToDeleteSegNo > 0 && OldestToKeepSegNo > 0)
{
GetXLogSegName(first_to_del_str, FirstToDeleteSegNo, xlog_seg_size);
GetXLogSegName(oldest_to_keep_str, OldestToKeepSegNo, xlog_seg_size);
GetXLogFileName(first_to_del_str, tlinfo->tli, FirstToDeleteSegNo, xlog_seg_size);
GetXLogFileName(oldest_to_keep_str, tlinfo->tli, OldestToKeepSegNo, xlog_seg_size);
elog(LOG, "On timeline %i first segment %s is greater than oldest segment to keep %s",
tlinfo->tli, first_to_del_str, oldest_to_keep_str);

View File

@ -43,8 +43,8 @@ typedef struct ShowArchiveRow
char tli[20];
char parent_tli[20];
char switchpoint[20];
char min_segno[20];
char max_segno[20];
char min_segno[XLOG_FNAME_LEN+1];
char max_segno[XLOG_FNAME_LEN+1];
char n_segments[20];
char size[20];
char zratio[20];
@ -748,7 +748,7 @@ static void
show_archive_plain(const char *instance_name, uint32 xlog_seg_size,
parray *tli_list, bool show_name)
{
char segno_tmp[20];
char segno_tmp[XLOG_FNAME_LEN];
parray *actual_tli_list = parray_new();
#define SHOW_ARCHIVE_FIELDS_COUNT 10
int i;
@ -807,14 +807,14 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size,
cur++;
/* Min Segno */
GetXLogSegName(segno_tmp, tlinfo->begin_segno, xlog_seg_size);
GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->begin_segno, xlog_seg_size);
snprintf(row->min_segno, lengthof(row->min_segno), "%s",segno_tmp);
widths[cur] = Max(widths[cur], strlen(row->min_segno));
cur++;
/* Max Segno */
GetXLogSegName(segno_tmp, tlinfo->end_segno, xlog_seg_size);
GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->end_segno, xlog_seg_size);
snprintf(row->max_segno, lengthof(row->max_segno), "%s", segno_tmp);
widths[cur] = Max(widths[cur], strlen(row->max_segno));
@ -939,7 +939,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size,
int i,j;
PQExpBuffer buf = &show_buf;
parray *actual_tli_list = parray_new();
char segno_tmp[20];
char segno_tmp[XLOG_FNAME_LEN+1];
if (!first_instance)
appendPQExpBufferChar(buf, ',');
@ -968,7 +968,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size,
for (i = parray_num(actual_tli_list) - 1; i >= 0; i--)
{
timelineInfo *tlinfo = (timelineInfo *) parray_get(actual_tli_list, i);
char tmp_buf[20];
char tmp_buf[XLOG_FNAME_LEN+1];
float zratio = 0;
if (i != (parray_num(actual_tli_list) - 1))
@ -986,11 +986,11 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size,
(uint32) (tlinfo->switchpoint >> 32), (uint32) tlinfo->switchpoint);
json_add_value(buf, "switchpoint", tmp_buf, json_level, true);
GetXLogSegName(segno_tmp, tlinfo->begin_segno, xlog_seg_size);
GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->begin_segno, xlog_seg_size);
snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp);
json_add_value(buf, "min-segno", tmp_buf, json_level, true);
GetXLogSegName(segno_tmp, tlinfo->end_segno, xlog_seg_size);
GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->end_segno, xlog_seg_size);
snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp);
json_add_value(buf, "max-segno", tmp_buf, json_level, true);
@ -1033,11 +1033,11 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size,
json_add(buf, JT_BEGIN_OBJECT, &json_level);
GetXLogSegName(segno_tmp, lost_segments->begin_segno, xlog_seg_size);
GetXLogFileName(segno_tmp, tlinfo->tli, lost_segments->begin_segno, xlog_seg_size);
snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp);
json_add_value(buf, "begin-segno", tmp_buf, json_level, true);
GetXLogSegName(segno_tmp, lost_segments->end_segno, xlog_seg_size);
GetXLogFileName(segno_tmp, tlinfo->tli, lost_segments->end_segno, xlog_seg_size);
snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp);
json_add_value(buf, "end-segno", tmp_buf, json_level, true);
json_add(buf, JT_END_OBJECT, &json_level);

View File

@ -1413,10 +1413,18 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
self.assertEqual(timeline_1['status'], "OK")
self.assertEqual(len(timeline_3['lost-segments']), 2)
self.assertEqual(timeline_3['lost-segments'][0]['begin-segno'], '0000000000000012')
self.assertEqual(timeline_3['lost-segments'][0]['end-segno'], '0000000000000013')
self.assertEqual(timeline_3['lost-segments'][1]['begin-segno'], '0000000000000017')
self.assertEqual(timeline_3['lost-segments'][1]['end-segno'], '0000000000000017')
self.assertEqual(
timeline_3['lost-segments'][0]['begin-segno'],
'000000030000000000000012')
self.assertEqual(
timeline_3['lost-segments'][0]['end-segno'],
'000000030000000000000013')
self.assertEqual(
timeline_3['lost-segments'][1]['begin-segno'],
'000000030000000000000017')
self.assertEqual(
timeline_3['lost-segments'][1]['end-segno'],
'000000030000000000000017')
self.assertEqual(len(timeline_6['backups']), 0)
self.assertEqual(len(timeline_5['backups']), 0)
@ -1487,7 +1495,9 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
# sanity
for timeline in timelines:
self.assertEqual(timeline['min-segno'], '0000000000000001')
self.assertEqual(
timeline['min-segno'],
'000000010000000000000001')
self.assertEqual(timeline['status'], 'OK')
self.del_test_dir(module_name, fname)
@ -1539,7 +1549,9 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
# sanity
for timeline in timelines:
self.assertEqual(timeline['min-segno'], '0000000000000002')
self.assertEqual(
timeline['min-segno'],
'000000010000000000000002')
self.assertEqual(timeline['status'], 'OK')
self.del_test_dir(module_name, fname)
@ -1690,8 +1702,67 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_hexadecimal_timeline(self):
"""
Check that timelines are correct.
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
backup_id = self.backup_node(backup_dir, 'node', node)
node.pgbench_init(scale=2)
# create timelines
for i in range(1, 13):
# print(i)
node.cleanup()
self.restore_node(
backup_dir, 'node', node,
options=['--recovery-target-timeline={0}'.format(i)])
node.slow_start()
node.pgbench_init(scale=2)
show = self.show_archive(backup_dir)
timelines = show[0]['timelines']
print(timelines[0])
tli13 = timelines[0]
self.assertEqual(
13,
tli13['tli'])
self.assertEqual(
12,
tli13['parent-tli'])
self.assertEqual(
backup_id,
tli13['closest-backup-id'])
self.assertEqual(
'0000000D000000000000001B',
tli13['max-segno'])
# Clean after yourself
self.del_test_dir(module_name, fname)
# important - switchpoint may be NullOffset LSN and not actually existing in archive to boot.
# so write validation code accordingly
# so write WAL validation code accordingly
# change wal-seg-size
#

View File

@ -766,8 +766,8 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
output)
self.assertIn(
'On timeline 1 WAL segments between 0000000000000001 '
'and 0000000000000003 can be removed',
'On timeline 1 WAL segments between 000000010000000000000001 '
'and 000000010000000000000003 can be removed',
output)
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4)
@ -791,8 +791,8 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
output)
self.assertIn(
'On timeline 1 WAL segments between 0000000000000001 '
'and 0000000000000003 will be removed',
'On timeline 1 WAL segments between 000000010000000000000001 '
'and 000000010000000000000003 will be removed',
output)
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1)

View File

@ -63,10 +63,10 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
if not wal_name.endswith(".backup"):
if self.archive_compress:
wal_name = wal_name[-19:]
wal_name = wal_name[-27:]
wal_name = wal_name[:-3]
else:
wal_name = wal_name[-16:]
wal_name = wal_name[-24:]
self.assertTrue(wal_name >= min_wal)
self.assertTrue(wal_name <= max_wal)
@ -2234,8 +2234,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
'--log-level-console=verbose'])
self.assertIn(
'INFO: On timeline 4 WAL segments between 0000000000000002 '
'and 0000000000000006 can be removed',
'INFO: On timeline 4 WAL segments between 000000040000000000000002 '
'and 000000040000000000000006 can be removed',
output)
self.assertIn(
@ -2259,8 +2259,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
options=['--delete-wal', '--log-level-console=verbose'])
self.assertIn(
'INFO: On timeline 4 WAL segments between 0000000000000002 '
'and 0000000000000006 will be removed',
'INFO: On timeline 4 WAL segments between 000000040000000000000002 '
'and 000000040000000000000006 will be removed',
output)
self.assertIn(
@ -2281,11 +2281,11 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(
show_tli4_before['min-segno'],
'0000000000000002')
'000000040000000000000002')
self.assertEqual(
show_tli4_after['min-segno'],
'0000000000000006')
'000000040000000000000006')
self.assertFalse(show_tli5_after)
@ -2461,8 +2461,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.assertIn(
'LOG: Archive backup {0} to stay consistent protect from '
'purge WAL interval between 0000000000000004 and 0000000000000004 '
'on timeline 1'.format(B1), output)
'purge WAL interval between 000000010000000000000004 '
'and 000000010000000000000004 on timeline 1'.format(B1), output)
start_lsn_B4 = self.show_pb(backup_dir, 'node', B4)['start-lsn']
self.assertIn(
@ -2471,13 +2471,13 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.assertIn(
'LOG: Timeline 3 to stay reachable from timeline 1 protect '
'from purge WAL interval between 0000000000000005 and '
'0000000000000008 on timeline 2', output)
'from purge WAL interval between 000000020000000000000005 and '
'000000020000000000000008 on timeline 2', output)
self.assertIn(
'LOG: Timeline 3 to stay reachable from timeline 1 protect '
'from purge WAL interval between 0000000000000004 and '
'0000000000000005 on timeline 1', output)
'from purge WAL interval between 000000010000000000000004 and '
'000000010000000000000005 on timeline 1', output)
show_tli1_before = self.show_archive(backup_dir, 'node', tli=1)
show_tli2_before = self.show_archive(backup_dir, 'node', tli=2)
@ -2509,11 +2509,11 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(
show_tli4_before['min-segno'],
'0000000000000002')
'000000040000000000000002')
self.assertEqual(
show_tli4_after['min-segno'],
'0000000000000006')
'000000040000000000000006')
self.assertFalse(show_tli5_after)
@ -2528,19 +2528,19 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(
show_tli1_after['lost-segments'][0]['begin-segno'],
'0000000000000006')
'000000010000000000000006')
self.assertEqual(
show_tli1_after['lost-segments'][0]['end-segno'],
'0000000000000009')
'000000010000000000000009')
self.assertEqual(
show_tli2_after['lost-segments'][0]['begin-segno'],
'0000000000000009')
'000000020000000000000009')
self.assertEqual(
show_tli2_after['lost-segments'][0]['end-segno'],
'0000000000000009')
'000000020000000000000009')
self.validate_pb(backup_dir, 'node')