db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-24 08:52:38 +02:00
Code Issues Releases Activity

Read recovery_time and recovery_xid from WAL segments whenever possible

Browse Source
This commit is contained in:
Artur Zakirov 2017-03-24 16:58:35 +03:00
parent 62953a7f81
commit b31485640b
9 changed files with 167 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
backup.c
View File

@ -67,7 +67,6 @@ static parray *do_backup_database(parray *backup_list, bool smooth_checkpoint);
static void pg_start_backup(const char *label, bool smooth, pgBackup *backup);
static void pg_stop_backup(pgBackup *backup);
static void pg_switch_xlog(void);
static bool pg_is_standby(void);
static void add_pgdata_files(parray *files, const char *root);
@ -169,7 +168,7 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
/* start stream replication */
if (stream_wal)
{
join_path_components(dst_backup_path, database_path, "pg_xlog");
join_path_components(dst_backup_path, database_path, PG_XLOG_DIR);
dir_create_dir(dst_backup_path, DIR_PERMISSION);
pthread_mutex_lock(&check_stream_mut);
@ -186,14 +185,14 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
char label_path[MAXPGPATH];
/* If backup_label does not exist in $PGDATA, stop taking backup */
join_path_components(label_path, pgdata, "backup_label");
join_path_components(label_path, pgdata, PG_BACKUP_LABEL_FILE);
/* Leave if no backup file */
if (!fileExists(label_path))
{
elog(LOG, "backup_label does not exist, stopping backup");
elog(LOG, "%s does not exist, stopping backup", PG_BACKUP_LABEL_FILE);
pg_stop_backup(NULL);
elog(ERROR, "backup_label does not exist in PGDATA.");
elog(ERROR, "%s does not exist in PGDATA", PG_BACKUP_LABEL_FILE);
}
}
@ -353,7 +352,7 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
/* Scan backup pg_xlog dir */
list_file = parray_new();
join_path_components(pg_xlog_path, database_path, "pg_xlog");
join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
dir_list_file(list_file, pg_xlog_path, false, true, false);
/* Remove file path root prefix and calc meta */
@ -739,6 +738,8 @@ wait_archive_lsn(XLogRecPtr lsn, bool prev_segno)
char wal_file[MAXFNAMELEN];
uint32 try_count = 0;
Assert(!stream_wal);
tli = get_current_timeline(false);
/* As well as WAL file name */
@ -764,8 +765,8 @@ wait_archive_lsn(XLogRecPtr lsn, bool prev_segno)
if (archive_timeout > 0 && try_count > archive_timeout)
elog(ERROR,
"switched WAL could not be archived in %d seconds",
archive_timeout);
"switched WAL segment %s could not be archived in %d seconds",
wal_file, archive_timeout);
}
}
@ -778,6 +779,8 @@ pg_stop_backup(pgBackup *backup)
PGresult *res;
uint32 xlogid;
uint32 xrecoff;
time_t recovery_time;
TransactionId recovery_xid;
/* Remove annoying NOTICE messages generated by backend */
res = pgut_execute(backup_conn, "SET client_min_messages = warning;",
@ -786,10 +789,18 @@ pg_stop_backup(pgBackup *backup)
if (from_replica)
res = pgut_execute(backup_conn,
"SELECT * FROM pg_stop_backup(false)", 0, NULL);
"SELECT *, txid_snapshot_xmax(txid_current_snapshot()) FROM pg_stop_backup(false)",
0, NULL);
else
res = pgut_execute(backup_conn,
"SELECT * FROM pg_stop_backup()", 0, NULL);
"SELECT *, txid_snapshot_xmax(txid_current_snapshot()) FROM pg_stop_backup()",
0, NULL);
/*
* We will use this value if there are no transactions between start_lsn
* and stop_lsn.
*/
recovery_time = time(NULL);
/*
* Extract timeline and LSN from results of pg_stop_backup()
@ -809,7 +820,7 @@ pg_stop_backup(pgBackup *backup)
Assert(PQnfields(res) >= 3);
pgBackupGetPath(&current, path, lengthof(path), DATABASE_DIR);
join_path_components(backup_label, path, "backup_label");
join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE);
/* Write backup_label */
fp = fopen(backup_label, "w");
@ -823,7 +834,7 @@ pg_stop_backup(pgBackup *backup)
file = pgFileNew(backup_label, true);
calc_file(file);
free(file->path);
file->path = strdup("backup_label");
file->path = strdup(PG_BACKUP_LABEL_FILE);
parray_append(backup_files_list, file);
/* Write tablespace_map */
@ -847,7 +858,17 @@ pg_stop_backup(pgBackup *backup)
file->path = strdup("tablespace_map");
parray_append(backup_files_list, file);
}
if (sscanf(PQgetvalue(res, 0, 3), XID_FMT, &recovery_xid) != 1)
elog(ERROR,
"result of txid_snapshot_xmax() is invalid: %s",
PQerrorMessage(backup_conn));
}
else
if (sscanf(PQgetvalue(res, 0, 1), XID_FMT, &recovery_xid) != 1)
elog(ERROR,
"result of txid_snapshot_xmax() is invalid: %s",
PQerrorMessage(backup_conn));
PQclear(res);
@ -857,63 +878,30 @@ pg_stop_backup(pgBackup *backup)
/* Fill in fields if backup exists */
if (backup != NULL)
{
char *xlog_path,
stream_xlog_path[MAXPGPATH];
if (stream_wal)
{
join_path_components(stream_xlog_path, pgdata, PG_XLOG_DIR);
xlog_path = stream_xlog_path;
}
else
xlog_path = arclog_path;
backup->tli = get_current_timeline(false);
backup->stop_lsn = stop_backup_lsn;
if (from_replica)
res = pgut_execute(backup_conn, TXID_CURRENT_IF_SQL, 0, NULL);
else
res = pgut_execute(backup_conn, TXID_CURRENT_SQL, 0, NULL);
if (sscanf(PQgetvalue(res, 0, 0), XID_FMT, &backup->recovery_xid) != 1)
elog(ERROR,
"result of txid_current() is invalid: %s",
PQerrorMessage(backup_conn));
backup->recovery_time = time(NULL);
elog(LOG, "finish backup: tli=%X lsn=%X/%08X xid=%s",
backup->tli,
(uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn,
PQgetvalue(res, 0, 0));
PQclear(res);
if (!read_recovery_info(xlog_path, backup->tli,
backup->start_lsn, backup->stop_lsn,
&backup->recovery_time, &backup->recovery_xid))
{
backup->recovery_time = recovery_time;
backup->recovery_xid = recovery_xid;
}
}
}
/*
* Switch to a new WAL segment for master.
*/
static void
pg_switch_xlog(void)
{
PGresult *res;
XLogRecPtr lsn;
uint32 xlogid;
uint32 xrecoff;
/* Remove annoying NOTICE messages generated by backend */
res = pgut_execute(backup_conn, "SET client_min_messages = warning;", 0,
NULL);
PQclear(res);
res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_xlog()",
0, NULL);
/*
* Extract timeline and LSN from results of pg_stop_backup()
* and friends.
*/
XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff);
/* Calculate LSN */
lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff;
PQclear(res);
/* Wait for returned lsn - 1 in archive folder */
wait_archive_lsn(lsn, false);
}
/*
* Check if node is a standby by looking at the presence of
* recovery.conf.
@ -957,11 +945,10 @@ backup_cleanup(bool fatal, void *userdata)
return;
/* If backup_label exist in $PGDATA, notify stop of backup to PostgreSQL */
snprintf(path, lengthof(path), "%s/backup_label", pgdata);
make_native_path(path);
join_path_components(path, pgdata, PG_BACKUP_LABEL_FILE);
if (fileExists(path))
{
elog(LOG, "backup_label exists, stop backup");
elog(LOG, "%s exists, stop backup", PG_BACKUP_LABEL_FILE);
pg_stop_backup(NULL); /* don't care stop_lsn on error case */
}
@ -1240,7 +1227,7 @@ add_pgdata_files(parray *files, const char *root)
relative = file->path + strlen(root) + 1;
if (!path_is_prefix_of_path("base", relative) &&
/*!path_is_prefix_of_path("global", relative) &&*/ //TODO What's wrong with this line?
!path_is_prefix_of_path("pg_tblspc", relative))
!path_is_prefix_of_path(PG_TBLSPC_DIR, relative))
continue;
/* Get file name from path */
@ -1508,7 +1495,7 @@ make_pagemap_from_ptrack(parray *files)
}
/* For unix only now */
sscanf(tmp_path, "%u/%u_ptrack", &db_oid, &rel_oid);
tablespace = strstr(p->ptrack_path, "pg_tblspc");
tablespace = strstr(p->ptrack_path, PG_TBLSPC_DIR);
if (tablespace != NULL)
sscanf(tablespace+10, "%i/", &tablespace_oid);

60
dir.c
View File

@ -27,43 +27,43 @@
*/
const char *pgdata_exclude_dir[] =
{
"pg_xlog",
/*
* Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
* when stats_temp_directory is set because PGSS_TEXT_FILE is always created
* there.
*/
"pg_stat_tmp",
"pgsql_tmp",
PG_XLOG_DIR,
/*
* Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
* when stats_temp_directory is set because PGSS_TEXT_FILE is always created
* there.
*/
"pg_stat_tmp",
"pgsql_tmp",
/*
* It is generally not useful to backup the contents of this directory even
* if the intention is to restore to another master. See backup.sgml for a
* more detailed description.
*/
"pg_replslot",
/*
* It is generally not useful to backup the contents of this directory even
* if the intention is to restore to another master. See backup.sgml for a
* more detailed description.
*/
"pg_replslot",
/* Contents removed on startup, see dsm_cleanup_for_mmap(). */
"pg_dynshmem",
/* Contents removed on startup, see dsm_cleanup_for_mmap(). */
"pg_dynshmem",
/* Contents removed on startup, see AsyncShmemInit(). */
"pg_notify",
/* Contents removed on startup, see AsyncShmemInit(). */
"pg_notify",
/*
* Old contents are loaded for possible debugging but are not required for
* normal operation, see OldSerXidInit().
*/
"pg_serial",
/*
* Old contents are loaded for possible debugging but are not required for
* normal operation, see OldSerXidInit().
*/
"pg_serial",
/* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
"pg_snapshots",
/* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
"pg_snapshots",
/* Contents zeroed on startup, see StartupSUBTRANS(). */
"pg_subtrans",
/* Contents zeroed on startup, see StartupSUBTRANS(). */
"pg_subtrans",
/* end of list */
NULL, /* pg_log will be set later */
NULL
/* end of list */
NULL, /* pg_log will be set later */
NULL
};
static char *pgdata_exclude_files[] =

3
init.c
View File

@ -63,7 +63,8 @@ do_init(void)
join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE);
fp = fopen(path, "wt");
if (fp == NULL)
elog(ERROR, "cannot create pg_probackup.conf: %s", strerror(errno));
elog(ERROR, "cannot create %s: %s",
BACKUP_CATALOG_CONF_FILE, strerror(errno));
fprintf(fp, "system-identifier = %li\n", _system_identifier);
fprintf(fp, "\n");

74
parsexlog.c
View File

@ -90,12 +90,11 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
if (errormsg)
elog(ERROR, "could not read WAL record at %X/%X: %s",
(uint32) (errptr >> 32), (uint32) (errptr),
errormsg);
(uint32) (errptr >> 32), (uint32) (errptr),
errormsg);
else
elog(ERROR, "could not read WAL record at %X/%X",
(uint32) (errptr >> 32),
(uint32) (errptr));
(uint32) (errptr >> 32), (uint32) (errptr));
}
extractPageInfo(xlogreader);
@ -145,7 +144,7 @@ validate_wal(pgBackup *backup,
while (true)
{
bool timestamp_record;
bool timestamp_record;
record = XLogReadRecord(xlogreader, startpoint, &errormsg);
if (record == NULL)
@ -250,6 +249,66 @@ validate_wal(pgBackup *backup,
}
}
/*
* Read from archived WAL segments latest recovery time and xid. All necessary
* segments present at archive folder. We waited **stop_lsn** in
* pg_stop_backup().
*/
bool
read_recovery_info(const char *archivedir, TimeLineID tli,
XLogRecPtr start_lsn, XLogRecPtr stop_lsn,
time_t *recovery_time, TransactionId *recovery_xid)
{
XLogRecPtr startpoint = stop_lsn;
XLogReaderState *xlogreader;
XLogPageReadPrivate private;
private.archivedir = archivedir;
private.tli = tli;
xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
if (xlogreader == NULL)
elog(ERROR, "out of memory");
/* Read records from stop_lsn down to start_lsn */
do
{
XLogRecord *record;
TimestampTz last_time = 0;
char *errormsg;
record = XLogReadRecord(xlogreader, startpoint, &errormsg);
if (record == NULL)
{
XLogRecPtr errptr;
errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
if (errormsg)
elog(ERROR, "could not read WAL record at %X/%X: %s",
(uint32) (errptr >> 32), (uint32) (errptr),
errormsg);
else
elog(ERROR, "could not read WAL record at %X/%X",
(uint32) (errptr >> 32), (uint32) (errptr));
}
/* Read previous record */
startpoint = record->xl_prev;
if (getRecordTimestamp(xlogreader, &last_time))
{
*recovery_time = timestamptz_to_time_t(last_time);
*recovery_xid = XLogRecGetXid(xlogreader);
return true;
}
} while (startpoint >= start_lsn);
/* Didn't find timestamp from WAL records between start_lsn and stop_lsn */
return false;
}
/* XLogreader callback function, to read a WAL page */
static int
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
@ -421,18 +480,19 @@ getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime)
*recordXtime = ((xl_restore_point *) XLogRecGetData(record))->rp_time;
return true;
}
if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_COMMIT ||
else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_COMMIT ||
xact_info == XLOG_XACT_COMMIT_PREPARED))
{
*recordXtime = ((xl_xact_commit *) XLogRecGetData(record))->xact_time;
return true;
}
if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_ABORT ||
else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_ABORT ||
xact_info == XLOG_XACT_ABORT_PREPARED))
{
*recordXtime = ((xl_xact_abort *) XLogRecGetData(record))->xact_time;
return true;
}
return false;
}

2
pg_probackup.c
View File

@ -16,7 +16,7 @@
#include <time.h>
#include <sys/stat.h>
const char *PROGRAM_VERSION = "1.1.2";
const char *PROGRAM_VERSION = "1.1.3";
const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup";
const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues";

5
pg_probackup.h
View File

@ -42,7 +42,6 @@
#define BACKUP_CONF_FILE "backup.conf"
#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf"
#define BACKUP_CATALOG_PID "pg_probackup.pid"
#define MKDIRS_SH_FILE "mkdirs.sh"
#define DATABASE_FILE_LIST "file_database.txt"
#define PG_BACKUP_LABEL_FILE "backup_label"
#define PG_BLACK_LIST "black_list"
@ -358,6 +357,10 @@ extern void validate_wal(pgBackup *backup,
time_t target_time,
TransactionId target_xid,
TimeLineID tli);
extern bool read_recovery_info(const char *archivedir, TimeLineID tli,
XLogRecPtr start_lsn, XLogRecPtr stop_lsn,
time_t *recovery_time,
TransactionId *recovery_xid);
/* in util.c */
extern TimeLineID get_current_timeline(bool safe);

9
restore.c
View File

@ -91,7 +91,6 @@ do_restore(time_t backup_id,
pgBackup *base_backup = NULL;
pgBackup *dest_backup = NULL;
pgRecoveryTarget *rt = NULL;
bool need_recovery_conf = true;
/* PGDATA and ARCLOG_PATH are always required */
if (pgdata == NULL)
@ -200,9 +199,6 @@ base_backup_found:
/* Tablespace directories checking */
check_tablespace_mapping((pgBackup *) parray_get(backups, last_diff_index));
if (dest_backup && dest_backup->stream)
need_recovery_conf = target_time != NULL || target_xid != NULL;
/* Restore backups from base_index to last_diff_index */
for (i = base_index; i >= last_diff_index; i--)
{
@ -216,9 +212,8 @@ base_backup_found:
}
/* create recovery.conf */
if (need_recovery_conf)
create_recovery_conf(backup_id, target_time, target_xid,
target_inclusive, base_backup->tli);
create_recovery_conf(backup_id, target_time, target_xid,
target_inclusive, base_backup->tli);
/* cleanup */
parray_walk(backups, pgBackupFree);

2
tests/expected/option_version.out
View File

@ -1 +1 @@
pg_probackup 1.1.2
pg_probackup 1.1.3

13
validate.c
View File

@ -38,8 +38,7 @@ do_validate(time_t backup_id,
pgRecoveryTarget *rt = NULL;
pgBackup *base_backup = NULL;
pgBackup *dest_backup = NULL;
bool success_validate,
need_validate_wal = true;
bool success_validate;
catalog_lock(false);
@ -128,9 +127,6 @@ base_backup_found:
Assert(last_diff_index <= base_index);
if (dest_backup && dest_backup->stream)
need_validate_wal = target_time != NULL || target_xid != NULL;
/* Validate backups from base_index to last_diff_index */
for (i = base_index; i >= last_diff_index; i--)
{
@ -142,9 +138,10 @@ base_backup_found:
success_validate;
}
/* and now we must check WALs */
if (need_validate_wal)
validate_wal((pgBackup *) parray_get(backups, last_diff_index),
/* And now we must check WALs */
dest_backup = (pgBackup *) parray_get(backups, last_diff_index);
if (!dest_backup->stream || (target_time != NULL || target_xid != NULL))
validate_wal(dest_backup,
arclog_path,
rt->recovery_target_time,
rt->recovery_target_xid,