db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-09 14:45:47 +02:00
Code Issues Releases Activity

Merge branch 'release_2_4' into issue_66

Browse Source
This commit is contained in:
Grigory Smolkin 2020-06-12 02:10:26 +03:00
commit 8b8a79f9dc
6 changed files with 58 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
src/backup.c
View File

@ -83,7 +83,7 @@ static void *backup_files(void *arg);
static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs);
static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, static void pg_start_backup(const char *label, bool smooth, pgBackup *backup,
PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn); PGNodeInfo *nodeInfo, PGconn *conn);
static void pg_switch_wal(PGconn *conn); static void pg_switch_wal(PGconn *conn);
static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo);
static int checkpoint_timeout(PGconn *backup_conn); static int checkpoint_timeout(PGconn *backup_conn);
@ -149,9 +149,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
parray *external_dirs = NULL; parray *external_dirs = NULL;
parray *database_map = NULL; parray *database_map = NULL;
PGconn *master_conn = NULL;
PGconn *pg_startbackup_conn = NULL;
/* used for multitimeline incremental backup */ /* used for multitimeline incremental backup */
parray *tli_list = NULL; parray *tli_list = NULL;
@ -168,6 +165,18 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
check_external_for_tablespaces(external_dirs, backup_conn); check_external_for_tablespaces(external_dirs, backup_conn);
} }
/* Clear ptrack files for not PTRACK backups */
if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable)
pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num);
/* notify start of backup to PostgreSQL server */
time2iso(label, lengthof(label), current.start_time);
strncat(label, " with pg_probackup", lengthof(label) -
strlen(" with pg_probackup"));
/* Call pg_start_backup function in PostgreSQL connect */
pg_start_backup(label, smooth_checkpoint, &current, nodeInfo, backup_conn);
/* Obtain current timeline */ /* Obtain current timeline */
#if PG_VERSION_NUM >= 90600 #if PG_VERSION_NUM >= 90600
current.tli = get_current_timeline(backup_conn); current.tli = get_current_timeline(backup_conn);
@ -175,6 +184,15 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
current.tli = get_current_timeline_from_control(false); current.tli = get_current_timeline_from_control(false);
#endif #endif
/* In PAGE mode or in ARCHIVE wal-mode wait for current segment */
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal)
/*
* Do not wait start_lsn for stream backup.
* Because WAL streaming will start after pg_start_backup() in stream
* mode.
*/
wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false);
/* /*
* In incremental backup mode ensure that already-validated * In incremental backup mode ensure that already-validated
* backup on current timeline exists and get its filelist. * backup on current timeline exists and get its filelist.
@ -252,29 +270,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
} }
} }
/* Clear ptrack files for FULL and PAGE backup */
if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable)
pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num);
/* notify start of backup to PostgreSQL server */
time2iso(label, lengthof(label), current.start_time);
strncat(label, " with pg_probackup", lengthof(label) -
strlen(" with pg_probackup"));
/* Create connection to master server needed to call pg_start_backup */
if (current.from_replica && exclusive_backup)
{
master_conn = pgut_connect(instance_config.master_conn_opt.pghost,
instance_config.master_conn_opt.pgport,
instance_config.master_conn_opt.pgdatabase,
instance_config.master_conn_opt.pguser);
pg_startbackup_conn = master_conn;
}
else
pg_startbackup_conn = backup_conn;
pg_start_backup(label, smooth_checkpoint, &current, nodeInfo, backup_conn, pg_startbackup_conn);
/* For incremental backup check that start_lsn is not from the past /* For incremental backup check that start_lsn is not from the past
* Though it will not save us if PostgreSQL instance is actually * Though it will not save us if PostgreSQL instance is actually
* restored STREAM backup. * restored STREAM backup.
@ -342,7 +337,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
* Get database_map (name to oid) for use in partial restore feature. * Get database_map (name to oid) for use in partial restore feature.
* It's possible that we fail and database_map will be NULL. * It's possible that we fail and database_map will be NULL.
*/ */
database_map = get_database_map(pg_startbackup_conn); database_map = get_database_map(backup_conn);
/* /*
* Append to backup list all files and directories * Append to backup list all files and directories
@ -571,7 +566,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
} }
/* Notify end of backup */ /* Notify end of backup */
pg_stop_backup(&current, pg_startbackup_conn, nodeInfo); pg_stop_backup(&current, backup_conn, nodeInfo);
/* In case of backup from replica >= 9.6 we must fix minRecPoint, /* In case of backup from replica >= 9.6 we must fix minRecPoint,
* First we must find pg_control in backup_files_list. * First we must find pg_control in backup_files_list.
@ -1101,19 +1096,15 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz)
*/ */
static void static void
pg_start_backup(const char *label, bool smooth, pgBackup *backup, pg_start_backup(const char *label, bool smooth, pgBackup *backup,
PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *pg_startbackup_conn) PGNodeInfo *nodeInfo, PGconn *conn)
{ {
PGresult *res; PGresult *res;
const char *params[2]; const char *params[2];
uint32 lsn_hi; uint32 lsn_hi;
uint32 lsn_lo; uint32 lsn_lo;
PGconn *conn;
params[0] = label; params[0] = label;
/* For 9.5 replica we call pg_start_backup() on master */
conn = pg_startbackup_conn;
/* 2nd argument is 'fast'*/ /* 2nd argument is 'fast'*/
params[1] = smooth ? "false" : "true"; params[1] = smooth ? "false" : "true";
if (!exclusive_backup) if (!exclusive_backup)
@ -1132,7 +1123,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
* is necessary to call pg_stop_backup() in backup_cleanup(). * is necessary to call pg_stop_backup() in backup_cleanup().
*/ */
backup_in_progress = true; backup_in_progress = true;
pgut_atexit_push(backup_stopbackup_callback, pg_startbackup_conn); pgut_atexit_push(backup_stopbackup_callback, conn);
/* Extract timeline and LSN from results of pg_start_backup() */ /* Extract timeline and LSN from results of pg_start_backup() */
XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo);
@ -1152,15 +1143,6 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
* (because in 9.5 only superuser can switch WAL) * (because in 9.5 only superuser can switch WAL)
*/ */
pg_switch_wal(conn); pg_switch_wal(conn);
/* In PAGE mode or in ARCHIVE wal-mode wait for current segment */
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal)
/*
* Do not wait start_lsn for stream backup.
* Because WAL streaming will start after pg_start_backup() in stream
* mode.
*/
wait_wal_lsn(backup->start_lsn, true, backup->tli, false, true, ERROR, false);
} }
/* /*

29
tests/archive.py
View File

@ -765,10 +765,6 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
before = master.safe_psql("postgres", "SELECT * FROM t_heap") before = master.safe_psql("postgres", "SELECT * FROM t_heap")
master.safe_psql(
"postgres",
"CHECKPOINT")
self.wait_until_replica_catch_with_master(master, replica) self.wait_until_replica_catch_with_master(master, replica)
backup_id = self.backup_node( backup_id = self.backup_node(
@ -864,10 +860,6 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
"md5(repeat(i::text,10))::tsvector as tsvector " "md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0, 60000) i") "from generate_series(0, 60000) i")
master.psql(
"postgres",
"CHECKPOINT")
backup_id = self.backup_node( backup_id = self.backup_node(
backup_dir, 'replica', replica, backup_dir, 'replica', replica,
options=[ options=[
@ -977,10 +969,6 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
replica.promote() replica.promote()
replica.safe_psql(
'postgres',
'CHECKPOINT')
master.pgbench_init(scale=10) master.pgbench_init(scale=10)
replica.pgbench_init(scale=10) replica.pgbench_init(scale=10)
@ -1221,11 +1209,6 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
# create timeline t2 # create timeline t2
replica.promote() replica.promote()
# do checkpoint to increment timeline ID in pg_control
replica.safe_psql(
'postgres',
'CHECKPOINT')
# FULL backup replica # FULL backup replica
A1 = self.backup_node( A1 = self.backup_node(
backup_dir, 'replica', replica) backup_dir, 'replica', replica)
@ -1959,7 +1942,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
replica.slow_start(replica=True) replica.slow_start(replica=True)
node.safe_psql('postgres', 'CHECKPOINT') # FULL
self.backup_node(backup_dir, 'replica', replica, options=['--stream'])
if self.get_version(replica) < 100000: if self.get_version(replica) < 100000:
pg_receivexlog_path = self.get_bin_path('pg_receivexlog') pg_receivexlog_path = self.get_bin_path('pg_receivexlog')
@ -1981,14 +1965,18 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
'Failed to start pg_receivexlog: {0}'.format( 'Failed to start pg_receivexlog: {0}'.format(
pg_receivexlog.communicate()[1])) pg_receivexlog.communicate()[1]))
replica.safe_psql(
'postgres',
'CHECKPOINT')
node.safe_psql( node.safe_psql(
"postgres", "postgres",
"create table t_heap as select i as id, md5(i::text) as text, " "create table t_heap as select i as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector " "md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1000000) i") "from generate_series(0,1000000) i")
# FULL # PAGE
self.backup_node(backup_dir, 'replica', replica, options=['--stream']) self.backup_node(backup_dir, 'replica', replica, backup_type='page')
node.safe_psql( node.safe_psql(
"postgres", "postgres",
@ -2027,6 +2015,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
pg_receivexlog.kill() pg_receivexlog.kill()
self.del_test_dir(module_name, fname) self.del_test_dir(module_name, fname)
@unittest.skip("skip")
def test_multi_timeline_recovery_prefetching(self): def test_multi_timeline_recovery_prefetching(self):
"""""" """"""
fname = self.id().split('.')[3] fname = self.id().split('.')[3]

28
tests/backup.py
View File

@ -324,6 +324,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node( node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'), base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True, set_replication=True,
ptrack_enable=True,
initdb_params=['--data-checksums']) initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@ -663,12 +664,10 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"\n Output: {0} \n CMD: {1}".format( "\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd)) repr(self.output), self.cmd))
except ProbackupException as e: except ProbackupException as e:
self.assertTrue( self.assertIn(
'WARNING: page verification failed, ' 'ERROR: Corruption detected in file "{0}", block 1: '
'calculated checksum' in e.message and 'page header invalid, pd_lower'.format(heap_fullpath),
'ERROR: query failed: ERROR: ' e.message,
'invalid page in block 1 of relation' in e.message and
'ERROR: Data files transferring failed' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format( '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd)) repr(e.message), self.cmd))
@ -806,12 +805,10 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"\n Output: {0} \n CMD: {1}".format( "\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd)) repr(self.output), self.cmd))
except ProbackupException as e: except ProbackupException as e:
self.assertTrue( self.assertIn(
'WARNING: page verification failed, ' 'ERROR: Corruption detected in file "{0}", block 1: '
'calculated checksum' in e.message and 'page header invalid, pd_lower'.format(heap_fullpath),
'ERROR: query failed: ERROR: ' e.message,
'invalid page in block 1 of relation' in e.message and
'ERROR: Data files transferring failed' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format( '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd)) repr(e.message), self.cmd))
@ -1480,7 +1477,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node( node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'), base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True, set_replication=True,
ptrack_enable=True, ptrack_enable=self.ptrack,
initdb_params=['--data-checksums']) initdb_params=['--data-checksums'])
self.init_pb(backup_dir) self.init_pb(backup_dir)
@ -2131,9 +2128,8 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"TO backup".format(fname)) "TO backup".format(fname))
else: else:
fnames = [ fnames = [
'pg_catalog.pg_ptrack_get_pagemapset(pg_lsn)', 'pg_catalog.ptrack_get_pagemapset(pg_lsn)',
'pg_catalog.pg_ptrack_control_lsn()', 'pg_catalog.ptrack_init_lsn()'
'pg_catalog.pg_ptrack_get_block(oid, oid, oid, bigint)'
] ]
for fname in fnames: for fname in fnames:

6
tests/ptrack.py
View File

@ -3946,9 +3946,9 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd) repr(self.output), self.cmd)
) )
except ProbackupException as e: except ProbackupException as e:
self.assertIn( self.assertTrue(
'ERROR: LSN from ptrack_control 0/0 differs from Start LSN of previous backup', 'ERROR: LSN from ptrack_control ' in e.message and
e.message, 'differs from Start LSN of previous backup' in e.message,
'\n Unexpected Error Message: {0}\n' '\n Unexpected Error Message: {0}\n'
' CMD: {1}'.format(repr(e.message), self.cmd)) ' CMD: {1}'.format(repr(e.message), self.cmd))

41
tests/replica.py
View File

@ -207,10 +207,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
before = master.safe_psql("postgres", "SELECT * FROM t_heap") before = master.safe_psql("postgres", "SELECT * FROM t_heap")
master.psql(
"postgres",
"CHECKPOINT")
self.wait_until_replica_catch_with_master(master, replica) self.wait_until_replica_catch_with_master(master, replica)
backup_id = self.backup_node( backup_id = self.backup_node(
@ -383,10 +379,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
"md5(repeat(i::text,10))::tsvector as tsvector " "md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,165000) i") "from generate_series(0,165000) i")
master.psql(
"postgres",
"CHECKPOINT")
master.psql( master.psql(
"postgres", "postgres",
"create table t_heap_1 as select i as id, md5(i::text) as text, " "create table t_heap_1 as select i as id, md5(i::text) as text, "
@ -726,9 +718,11 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
log_content) log_content)
self.assertIn( self.assertIn(
'LOG: stop_lsn: 0/4000028', 'LOG: stop_lsn: 0/4000000',
log_content) log_content)
self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE')
# Clean after yourself # Clean after yourself
self.del_test_dir(module_name, fname) self.del_test_dir(module_name, fname)
@ -1118,18 +1112,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
replica.promote() replica.promote()
replica.safe_psql(
'postgres',
'CHECKPOINT')
# replica.safe_psql(
# 'postgres',
# 'create table t2()')
#
# replica.safe_psql(
# 'postgres',
# 'CHECKPOINT')
self.backup_node( self.backup_node(
backup_dir, 'master', replica, data_dir=replica.data_dir, backup_dir, 'master', replica, data_dir=replica.data_dir,
backup_type='page') backup_type='page')
@ -1176,10 +1158,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
self.add_instance(backup_dir, 'replica', replica) self.add_instance(backup_dir, 'replica', replica)
replica.safe_psql(
'postgres',
'CHECKPOINT')
full_id = self.backup_node( full_id = self.backup_node(
backup_dir, 'replica', backup_dir, 'replica',
replica, options=['--stream']) replica, options=['--stream'])
@ -1191,20 +1169,12 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
'FROM generate_series(0,20) i') 'FROM generate_series(0,20) i')
self.wait_until_replica_catch_with_master(master, replica) self.wait_until_replica_catch_with_master(master, replica)
replica.safe_psql(
'postgres',
'CHECKPOINT')
self.backup_node( self.backup_node(
backup_dir, 'replica', replica, backup_dir, 'replica', replica,
backup_type='delta', options=['--stream']) backup_type='delta', options=['--stream'])
replica.promote() replica.promote()
replica.safe_psql(
'postgres',
'CHECKPOINT')
# failing, because without archving, it is impossible to # failing, because without archving, it is impossible to
# take multi-timeline backup. # take multi-timeline backup.
try: try:
@ -1297,7 +1267,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
# node2 is now master # node2 is now master
node2.promote() node2.promote()
node2.safe_psql('postgres', 'CHECKPOINT')
node2.safe_psql( node2.safe_psql(
'postgres', 'postgres',
@ -1331,7 +1300,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
# node1 is back to be a master # node1 is back to be a master
node1.promote() node1.promote()
node1.safe_psql('postgres', 'CHECKPOINT')
sleep(5) sleep(5)
@ -1420,7 +1388,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
# node2 is now master # node2 is now master
node2.promote() node2.promote()
node2.safe_psql('postgres', 'CHECKPOINT')
node2.safe_psql( node2.safe_psql(
'postgres', 'postgres',
@ -1454,7 +1421,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
# node1 is back to be a master # node1 is back to be a master
node1.promote() node1.promote()
node1.safe_psql('postgres', 'CHECKPOINT')
self.switch_wal_segment(node1) self.switch_wal_segment(node1)
sleep(5) sleep(5)
@ -1532,7 +1498,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
backup_type='delta', options=['--stream']) backup_type='delta', options=['--stream'])
replica.promote() replica.promote()
replica.safe_psql('postgres', 'CHECKPOINT')
# failing, because without archving, it is impossible to # failing, because without archving, it is impossible to
# take multi-timeline backup. # take multi-timeline backup.

23
tests/restore.py
View File

@ -980,9 +980,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# Create tablespace table # Create tablespace table
with node.connect("postgres") as con: with node.connect("postgres") as con:
con.connection.autocommit = True # con.connection.autocommit = True
con.execute("CHECKPOINT") # con.execute("CHECKPOINT")
con.connection.autocommit = False # con.connection.autocommit = False
con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc")
con.execute( con.execute(
"INSERT INTO tbl1 SELECT * " "INSERT INTO tbl1 SELECT * "
@ -1389,10 +1389,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
'postgres', 'postgres',
'create extension pageinspect') 'create extension pageinspect')
node.safe_psql(
'postgres',
'checkpoint')
node.safe_psql( node.safe_psql(
'postgres', 'postgres',
'insert into tbl select i from generate_series(0,100) as i') 'insert into tbl select i from generate_series(0,100) as i')
@ -3025,6 +3021,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node( node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'), base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True, set_replication=True,
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'], initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'}) pg_options={'autovacuum': 'off'})
@ -3276,17 +3273,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
self.backup_node( self.backup_node(
backup_dir, 'node', node, options=['--stream']) backup_dir, 'node', node, options=['--stream'])
node.pgbench_init(scale=1) node.pgbench_init(scale=5)
node.safe_psql(
'postgres',
'CHECKPOINT')
node.pgbench_init(scale=1)
node.safe_psql(
'postgres',
'CHECKPOINT')
node.safe_psql( node.safe_psql(
'postgres', 'postgres',