From 308d00b80c4f179b4807216538bd53b4e0044f28 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 13 Dec 2013 03:55:39 +0900 Subject: [PATCH] Obtain timeline ID with control file and not XLOG system function The system function used up to now was pg_xlogfile_name_offset, which cannot be used on a node in recovery, and it was the only way present to fetch the timeline ID of a backup, either incremental or full. So instead scan the control file of server and fetch the timeline from that. This also removes the restriction on which a backup could not be taken on a standby node. The next step being to have the possibility to take backups from streams. --- Makefile | 1 + backup.c | 96 +++++++++++++++++++++++++++++++++++++++++++++---------- fetch.c | 69 +++++++++++++++++++++++++++++++++++++++ pg_rman.h | 12 +++++++ 4 files changed, 161 insertions(+), 17 deletions(-) create mode 100644 fetch.c diff --git a/Makefile b/Makefile index 908ba0e3..2a10cdf7 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ SRCS = \ data.c \ delete.c \ dir.c \ + fetch.c \ init.c \ parray.c \ pg_rman.c \ diff --git a/backup.c b/backup.c index 130549f1..86ea2964 100644 --- a/backup.c +++ b/backup.c @@ -17,6 +17,7 @@ #include #include +#include "catalog/pg_control.h" #include "libpq/pqsignal.h" #include "pgut/pgut-port.h" @@ -48,7 +49,7 @@ static void confirm_block_size(const char *name, int blcksz); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); static void pg_stop_backup(pgBackup *backup); static void pg_switch_xlog(pgBackup *backup); -static void get_lsn(PGresult *res, TimeLineID *timeline, XLogRecPtr *lsn); +static void get_lsn(PGresult *res, XLogRecPtr *lsn); static void get_xid(PGresult *res, uint32 *xid); static bool execute_restartpoint(pgBackupOption bkupopt); @@ -60,6 +61,7 @@ static bool dirExists(const char *path); static void add_files(parray *files, const char *root, bool add_root, bool is_pgdata); static int strCompare(const void *str1, const void *str2); static void create_file_list(parray *files, const char *root, const char *prefix, bool is_append); +static TimeLineID get_current_timeline(void); /* * Take a backup of database. @@ -108,6 +110,13 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt) current.total_data_bytes = 0; current.read_data_bytes = 0; + /* + * Obtain current timeline by scanning control file, theh LSN + * obtained at output of pg_start_backup or pg_stop_backup does + * not contain this information. + */ + current.tli = get_current_timeline(); + /* notify start of backup to PostgreSQL server */ time2iso(label, lengthof(label), current.start_time); strncat(label, " with pg_rman", lengthof(label)); @@ -492,8 +501,8 @@ do_backup_arclog(parray *backup_list) pg_switch_xlog(¤t); /* - * To take incremental backup, the file list of the last completed database - * backup is needed. + * To take incremental backup, the file list of the last completed + * database backup is needed. */ prev_backup = catalog_get_last_arclog_backup(backup_list); if (verbose && prev_backup == NULL) @@ -985,10 +994,10 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; - res = execute("SELECT * from pg_xlogfile_name_offset(pg_start_backup($1, $2))", 2, params); + res = execute("SELECT pg_start_backup($1, $2)", 2, params); if (backup != NULL) - get_lsn(res, &backup->tli, &backup->start_lsn); + get_lsn(res, &backup->start_lsn); PQclear(res); disconnect(); } @@ -998,22 +1007,42 @@ wait_for_archive(pgBackup *backup, const char *sql) { PGresult *res; char ready_path[MAXPGPATH]; + char file_name[MAXFNAMELEN]; int try_count; + XLogRecPtr lsn; + TimeLineID tli; reconnect(); res = execute(sql, 0, NULL); + + /* Get LSN from execution result */ + get_lsn(res, &lsn); + + /* + * Enforce TLI obtention if backup is not present as this code + * path can be taken as a callback at exit. + */ + if (backup != NULL) + tli = backup->tli; + else + tli = get_current_timeline(); + + /* Fill in fields if backup exists */ if (backup != NULL) { - get_lsn(res, &backup->tli, &backup->stop_lsn); + backup->stop_lsn = lsn; elog(LOG, _("%s(): tli=%X lsn=%X/%08X"), __FUNCTION__, backup->tli, (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); } - /* get filename from the result of pg_xlogfile_name_offset() */ + /* As well as WAL file name */ + XLogFileName(file_name, tli, lsn); + snprintf(ready_path, lengthof(ready_path), - "%s/pg_xlog/archive_status/%s.ready", pgdata, PQgetvalue(res, 0, 0)); + "%s/pg_xlog/archive_status/%s.ready", pgdata, + file_name); elog(LOG, "%s() wait for %s", __FUNCTION__, ready_path); PQclear(res); @@ -1049,32 +1078,41 @@ static void pg_stop_backup(pgBackup *backup) { wait_for_archive(backup, - "SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup())"); + "SELECT * FROM pg_stop_backup()"); } /* - * Force switch to a new transaction log file and update backup->tli. + * Force switch to a new transaction log file */ static void pg_switch_xlog(pgBackup *backup) { wait_for_archive(backup, - "SELECT * FROM pg_xlogfile_name_offset(pg_switch_xlog())"); + "SELECT * FROM pg_switch_xlog())"); } /* - * Get TimeLineID and LSN from result of pg_xlogfile_name_offset(). + * Get LSN from result of pg_start_backup() or pg_stop_backup(). */ static void -get_lsn(PGresult *res, TimeLineID *timeline, XLogRecPtr *lsn) +get_lsn(PGresult *res, XLogRecPtr *lsn) { - if (res == NULL || PQntuples(res) != 1 || PQnfields(res) != 2) + uint32 xlogid; + uint32 xrecoff; + + if (res == NULL || PQntuples(res) != 1 || PQnfields(res) != 1) elog(ERROR_PG_COMMAND, - _("result of pg_xlogfile_name_offset() is invalid: %s"), + _("result of backup command is invalid: %s"), PQerrorMessage(connection)); - /* Extract timeline and LSN from result of pg_stop_backup() */ - XLogFromFileName(PQgetvalue(res, 0, 0), timeline, lsn); + /* + * Extract timeline and LSN from results of pg_stop_backup() + * and friends. + */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + + /* Calculate LSN */ + *lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; } /* @@ -1598,3 +1636,27 @@ create_file_list(parray *files, const char *root, const char *prefix, bool is_ap fclose(fp); } } + +/* + * Scan control file of given cluster at obtain the current timeline + * since last checkpoint that occurred on it. + */ +static TimeLineID +get_current_timeline(void) +{ + char *buffer; + size_t size; + ControlFileData control_file; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size); + + /* .. Then interpret it */ + if (size != PG_CONTROL_SIZE) + elog(ERROR_CORRUPTED, "unexpected control file size %d, expected %d\n", + (int) size, PG_CONTROL_SIZE); + memcpy(&control_file, buffer, sizeof(ControlFileData)); + + /* Finally return the timeline wanted */ + return control_file.checkPointCopy.ThisTimeLineID; +} diff --git a/fetch.c b/fetch.c new file mode 100644 index 00000000..922e4b0d --- /dev/null +++ b/fetch.c @@ -0,0 +1,69 @@ +/*------------------------------------------------------------------------- + * + * fetch.c + * Functions for fetching files from PostgreSQL data directory + * + * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "catalog/catalog.h" + +#include +#include +#include +#include +#include +#include + +#include "pg_rman.h" + +/* + * Read a file into memory. The file to be read is /. + * The file contents are returned in a malloc'd buffer, and *filesize + * is set to the length of the file. + * + * The returned buffer is always zero-terminated; the size of the returned + * buffer is actually *filesize + 1. That's handy when reading a text file. + * This function can be used to read binary files as well, you can just + * ignore the zero-terminator in that case. + * + */ +char * +slurpFile(const char *datadir, const char *path, size_t *filesize) +{ + int fd; + char *buffer; + struct stat statbuf; + char fullpath[MAXPGPATH]; + int len; + snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); + + if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1) + elog(ERROR_CORRUPTED, _("could not open file \"%s\" for reading: %s\n"), + fullpath, strerror(errno)); + + if (fstat(fd, &statbuf) < 0) + elog(ERROR_CORRUPTED, _("could not open file \"%s\" for reading: %s\n"), + fullpath, strerror(errno)); + + len = statbuf.st_size; + + buffer = pg_malloc(len + 1); + + if (read(fd, buffer, len) != len) + elog(ERROR_CORRUPTED, _("could not read file \"%s\": %s\n"), + fullpath, strerror(errno)); + + close(fd); + + /* Zero-terminate the buffer. */ + buffer[len] = '\0'; + + if (filesize) + *filesize = len; + return buffer; +} diff --git a/pg_rman.h b/pg_rman.h index a7f68810..8c09be77 100644 --- a/pg_rman.h +++ b/pg_rman.h @@ -214,6 +214,13 @@ typedef enum CompressionMode #define JoinPathEnd(str, prefix) \ ((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1) +/* + * Return timeline, xlog ID and record offset from an LSN of the type + * 0/B000188, usual result from pg_stop_backup() and friends. + */ +#define XLogDataFromLSN(data, xlogid, xrecoff) \ + sscanf(data, "%X/%X", xlogid, xrecoff) + /* path configuration */ extern char *backup_path; extern char *pgdata; @@ -253,6 +260,11 @@ extern int do_show(pgBackupRange *range, bool show_all); extern int do_delete(pgBackupRange *range, bool force); extern void pgBackupDelete(int keep_generations, int keep_days); +/* in fetch.c */ +extern char *slurpFile(const char *datadir, + const char *path, + size_t *filesize); + /* in validate.c */ extern int do_validate(pgBackupRange *range); extern void pgBackupValidate(pgBackup *backup, bool size_only, bool for_get_timeline, bool with_database);