From 78adfdfa3b7a846dfa3bf2e2fd0d8fc69fdd45b8 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 9 Oct 2018 12:51:41 +0300 Subject: [PATCH] Port to PostgreSQL 11 --- Makefile | 4 +- src/backup.c | 25 +++++++----- src/catalog.c | 14 +++---- src/configure.c | 53 ++++++++++++++++++++++++- src/delete.c | 25 +++++++----- src/init.c | 5 ++- src/merge.c | 7 ++-- src/parsexlog.c | 99 ++++++++++++++++++++++++++++------------------ src/pg_probackup.c | 31 ++++++++++++++- src/pg_probackup.h | 50 +++++++++++++++++------ src/restore.c | 4 +- src/util.c | 24 +++++++++++ src/utils/pgut.c | 9 +---- src/utils/pgut.h | 3 +- src/validate.c | 7 +++- 15 files changed, 264 insertions(+), 96 deletions(-) diff --git a/Makefile b/Makefile index e22bc86c..cddb92aa 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ else srchome=$(top_srcdir) endif -ifeq ($(MAJORVERSION),10) +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) OBJS += src/walmethods.o EXTRA_CLEAN += src/walmethods.c src/walmethods.h INCLUDES += src/walmethods.h @@ -64,7 +64,7 @@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ -ifeq ($(MAJORVERSION),10) +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h diff --git a/src/backup.c b/src/backup.c index 8e75f51c..067f44a9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -650,8 +650,9 @@ do_backup_instance(void) * reading WAL segments present in archives up to the point * where this backup has started. */ - extractPageMap(arclog_path, prev_backup->start_lsn, current.tli, - current.start_lsn, backup_files_list); + extractPageMap(arclog_path, current.tli, xlog_seg_size, + prev_backup->start_lsn, current.start_lsn, + backup_files_list); } else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { @@ -827,6 +828,11 @@ do_backup(time_t start_time) current.primary_conninfo = pgut_get_conninfo_string(backup_conn); +#if PG_VERSION_NUM >= 110000 + if (!RetrieveWalSegSize(backup_conn)) + elog(ERROR, "Failed to retreive wal_segment_size"); +#endif + current.compress_alg = compress_alg; current.compress_level = compress_level; @@ -918,8 +924,9 @@ do_backup(time_t start_time) /* compute size of wal files of this backup stored in the archive */ if (!current.stream) { - current.wal_bytes = XLOG_SEG_SIZE * - (current.stop_lsn/XLogSegSize - current.start_lsn/XLogSegSize + 1); + current.wal_bytes = xlog_seg_size * + (current.stop_lsn / xlog_seg_size - + current.start_lsn / xlog_seg_size + 1); } /* Backup is done. Update backup status */ @@ -1467,10 +1474,10 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) tli = get_current_timeline(false); /* Compute the name of the WAL file containig requested LSN */ - XLByteToSeg(lsn, targetSegNo); + GetXLogSegNo(lsn, targetSegNo, xlog_seg_size); if (wait_prev_segment) targetSegNo--; - XLogFileName(wal_segment, tli, targetSegNo); + GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size); /* * In pg_start_backup we wait for 'lsn' in 'pg_wal' directory iff it is @@ -1536,7 +1543,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) /* * A WAL segment found. Check LSN on it. */ - if (wal_contains_lsn(wal_segment_dir, lsn, tli)) + if (wal_contains_lsn(wal_segment_dir, lsn, tli, xlog_seg_size)) /* Target LSN was found */ { elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); @@ -1946,7 +1953,7 @@ pg_stop_backup(pgBackup *backup) elog(LOG, "Getting the Recovery Time from WAL"); - if (!read_recovery_info(xlog_path, backup->tli, + if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, backup->start_lsn, backup->stop_lsn, &backup->recovery_time, &backup->recovery_xid)) { @@ -2553,7 +2560,7 @@ StreamLog(void *arg) /* * Always start streaming at the beginning of a segment */ - startpos -= startpos % XLOG_SEG_SIZE; + startpos -= startpos % xlog_seg_size; /* Initialize timeout */ stream_stop_timeout = 0; diff --git a/src/catalog.c b/src/catalog.c index e7389d3d..e1fd2498 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -572,23 +572,23 @@ readBackupControlFile(const char *path) pgBackupInit(backup); if (access(path, F_OK) != 0) { - elog(WARNING, "control file \"%s\" doesn't exist", path); + elog(WARNING, "Control file \"%s\" doesn't exist", path); pgBackupFree(backup); return NULL; } - parsed_options = pgut_readopt(path, options, WARNING); + parsed_options = pgut_readopt(path, options, WARNING, true); if (parsed_options == 0) { - elog(WARNING, "control file \"%s\" is empty", path); + elog(WARNING, "Control file \"%s\" is empty", path); pgBackupFree(backup); return NULL; } if (backup->start_time == 0) { - elog(WARNING, "invalid ID/start-time, control file \"%s\" is corrupted", path); + elog(WARNING, "Invalid ID/start-time, control file \"%s\" is corrupted", path); pgBackupFree(backup); return NULL; } @@ -607,7 +607,7 @@ readBackupControlFile(const char *path) if (sscanf(start_lsn, "%X/%X", &xlogid, &xrecoff) == 2) backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; else - elog(WARNING, "invalid START_LSN \"%s\"", start_lsn); + elog(WARNING, "Invalid START_LSN \"%s\"", start_lsn); free(start_lsn); } @@ -619,7 +619,7 @@ readBackupControlFile(const char *path) if (sscanf(stop_lsn, "%X/%X", &xlogid, &xrecoff) == 2) backup->stop_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; else - elog(WARNING, "invalid STOP_LSN \"%s\"", stop_lsn); + elog(WARNING, "Invalid STOP_LSN \"%s\"", stop_lsn); free(stop_lsn); } @@ -644,7 +644,7 @@ readBackupControlFile(const char *path) else if (strcmp(status, "CORRUPT") == 0) backup->status = BACKUP_STATUS_CORRUPT; else - elog(WARNING, "invalid STATUS \"%s\"", status); + elog(WARNING, "Invalid STATUS \"%s\"", status); free(status); } diff --git a/src/configure.c b/src/configure.c index fe427833..28b55fd5 100644 --- a/src/configure.c +++ b/src/configure.c @@ -102,6 +102,13 @@ void pgBackupConfigInit(pgBackupConfig *config) { config->system_identifier = 0; + +#if PG_VERSION_NUM >= 110000 + config->xlog_seg_size = 0; +#else + config->xlog_seg_size = XLOG_SEG_SIZE; +#endif + config->pgdata = NULL; config->pgdatabase = NULL; config->pghost = NULL; @@ -140,6 +147,9 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) fprintf(out, "#Backup instance info\n"); fprintf(out, "PGDATA = %s\n", config->pgdata); fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); +#if PG_VERSION_NUM >= 110000 + fprintf(out, "xlog-seg-size = %u\n", config->xlog_seg_size); +#endif fprintf(out, "#Connection parameters:\n"); if (config->pgdatabase) @@ -253,6 +263,9 @@ readBackupCatalogConfigFile(void) { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, /* other options */ { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, +#if PG_VERSION_NUM >= 110000 + {'u', 0, "xlog-seg-size", &config->xlog_seg_size, SOURCE_FILE_STRICT}, +#endif /* archive options */ { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, {0} @@ -263,11 +276,44 @@ readBackupCatalogConfigFile(void) join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); pgBackupConfigInit(config); - pgut_readopt(path, options, ERROR); + pgut_readopt(path, options, ERROR, true); + +#if PG_VERSION_NUM >= 110000 + if (!IsValidWalSegSize(config->xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", config->xlog_seg_size); +#endif return config; } +/* + * Read xlog-seg-size from BACKUP_CATALOG_CONF_FILE. + */ +uint32 +get_config_xlog_seg_size(void) +{ +#if PG_VERSION_NUM >= 110000 + char path[MAXPGPATH]; + uint32 seg_size; + pgut_option options[] = + { + {'u', 0, "xlog-seg-size", &seg_size, SOURCE_FILE_STRICT}, + {0} + }; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, false); + + if (!IsValidWalSegSize(seg_size)) + elog(ERROR, "Invalid WAL segment size %u", seg_size); + + return seg_size; + +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + static void opt_log_level_console(pgut_option *opt, const char *arg) { @@ -349,6 +395,11 @@ show_configure_json(pgBackupConfig *config) json_add_key(buf, "system-identifier", json_level, true); appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); +#if PG_VERSION_NUM >= 110000 + json_add_key(buf, "xlog-seg-size", json_level, true); + appendPQExpBuffer(buf, "%u", config->xlog_seg_size); +#endif + /* Connection parameters */ if (config->pgdatabase) json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); diff --git a/src/delete.c b/src/delete.c index 0829e725..de29d2cf 100644 --- a/src/delete.c +++ b/src/delete.c @@ -15,7 +15,8 @@ #include static int pgBackupDeleteFiles(pgBackup *backup); -static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli); +static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size); int do_delete(time_t backup_id) @@ -23,8 +24,8 @@ do_delete(time_t backup_id) int i; parray *backup_list, *delete_list; + pgBackup *target_backup = NULL; time_t parent_id = 0; - bool backup_found = false; XLogRecPtr oldest_lsn = InvalidXLogRecPtr; TimeLineID oldest_tli = 0; @@ -56,9 +57,9 @@ do_delete(time_t backup_id) /* Save backup id to retreive increment backups */ parent_id = backup->start_time; - backup_found = true; + target_backup = backup; } - else if (backup_found) + else if (target_backup) { if (backup->backup_mode != BACKUP_MODE_FULL && backup->parent_backup == parent_id) @@ -93,6 +94,8 @@ do_delete(time_t backup_id) /* Clean WAL segments */ if (delete_wal) { + Assert(target_backup); + /* Find oldest LSN, used by backups */ for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) { @@ -106,7 +109,7 @@ do_delete(time_t backup_id) } } - delete_walfiles(oldest_lsn, oldest_tli); + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); } /* cleanup */ @@ -225,7 +228,7 @@ do_retention_purge(void) /* Purge WAL files */ if (delete_wal) { - delete_walfiles(oldest_lsn, oldest_tli); + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); } /* Cleanup */ @@ -313,7 +316,8 @@ pgBackupDeleteFiles(pgBackup *backup) * oldest_lsn. */ static void -delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli) +delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size) { XLogSegNo targetSegNo; char oldestSegmentNeeded[MAXFNAMELEN]; @@ -329,8 +333,9 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli) if (!XLogRecPtrIsInvalid(oldest_lsn)) { - XLByteToSeg(oldest_lsn, targetSegNo); - XLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo); + GetXLogSegNo(oldest_lsn, targetSegNo, xlog_seg_size); + GetXLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo, + xlog_seg_size); elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded); } @@ -436,7 +441,7 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - delete_walfiles(InvalidXLogRecPtr, 0); + delete_walfiles(InvalidXLogRecPtr, 0, xlog_seg_size); /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); diff --git a/src/init.c b/src/init.c index 712cba11..cd559cb4 100644 --- a/src/init.c +++ b/src/init.c @@ -54,7 +54,7 @@ do_add_instance(void) { char path[MAXPGPATH]; char arclog_path_dir[MAXPGPATH]; - struct stat st; + struct stat st; pgBackupConfig *config = pgut_new(pgBackupConfig); /* PGDATA is always required */ @@ -64,6 +64,8 @@ do_add_instance(void) /* Read system_identifier from PGDATA */ system_identifier = get_system_identifier(pgdata); + /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ + xlog_seg_size = get_xlog_seg_size(pgdata); /* Ensure that all root directories already exist */ if (access(backup_path, F_OK) != 0) @@ -97,6 +99,7 @@ do_add_instance(void) */ pgBackupConfigInit(config); config->system_identifier = system_identifier; + config->xlog_seg_size = xlog_seg_size; config->pgdata = pgdata; writeBackupCatalogConfigFile(config); diff --git a/src/merge.c b/src/merge.c index 9b8a54c1..260148e3 100644 --- a/src/merge.c +++ b/src/merge.c @@ -320,9 +320,10 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->data_bytes += file->write_size; } /* compute size of wal files of this backup stored in the archive */ - if (!current.stream) - to_backup->wal_bytes = XLOG_SEG_SIZE * - (to_backup->stop_lsn / XLogSegSize - to_backup->start_lsn / XLogSegSize + 1); + if (!to_backup->stream) + to_backup->wal_bytes = xlog_seg_size * + (to_backup->stop_lsn / xlog_seg_size - + to_backup->start_lsn / xlog_seg_size + 1); else to_backup->wal_bytes = BYTES_INVALID; diff --git a/src/parsexlog.c b/src/parsexlog.c index 34b4fa94..6f93bdd5 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -89,6 +89,7 @@ typedef struct XLogPageReadPrivate int thread_num; const char *archivedir; TimeLineID tli; + uint32 xlog_seg_size; bool manual_switch; bool need_switch; @@ -126,7 +127,8 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, TimeLineID *pageTLI); static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, - TimeLineID tli, bool allocate_reader); + TimeLineID tli, uint32 xlog_seg_size, + bool allocate_reader); static void CleanupXLogPageRead(XLogReaderState *xlogreader); static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel); @@ -160,7 +162,8 @@ switchToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) return false; /* Adjust next record position */ - XLogSegNoOffsetToRecPtr(private_data->xlogsegno, 0, arg->startpoint); + GetXLogRecPtr(private_data->xlogsegno, 0, + private_data->xlog_seg_size, arg->startpoint); /* We need to close previously opened file if it wasn't closed earlier */ CleanupXLogPageRead(xlogreader); /* Skip over the page header and contrecord if any */ @@ -200,7 +203,12 @@ doExtractPageMap(void *arg) char *errormsg; private_data = &extract_arg->private_data; +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(private_data->xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif if (xlogreader == NULL) elog(ERROR, "Thread [%d]: out of memory", private_data->thread_num); xlogreader->system_identifier = system_identifier; @@ -235,7 +243,7 @@ doExtractPageMap(void *arg) if (interrupted) elog(ERROR, "Thread [%d]: Interrupted during WAL reading", - private_data->thread_num); + private_data->thread_num); /* * We need to switch to the next WAL segment after reading previous @@ -292,7 +300,8 @@ doExtractPageMap(void *arg) /* continue reading at next record */ extract_arg->startpoint = InvalidXLogRecPtr; - XLByteToSeg(xlogreader->EndRecPtr, nextSegNo); + GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, + private_data->xlog_seg_size); } while (nextSegNo <= extract_arg->endSegNo && xlogreader->ReadRecPtr < extract_arg->endpoint); @@ -312,8 +321,8 @@ doExtractPageMap(void *arg) * file. */ void -extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, - XLogRecPtr endpoint, parray *files) +extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, parray *files) { int i; int threads_need = 0; @@ -333,7 +342,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, elog(ERROR, "Invalid endpoint value %X/%X", (uint32) (endpoint >> 32), (uint32) (endpoint)); - XLByteToSeg(endpoint, endSegNo); + GetXLogSegNo(endpoint, endSegNo, seg_size); nextSegNoToRead = 0; time(&start_time); @@ -349,7 +358,8 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, */ for (i = 0; i < num_threads; i++) { - InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, false); + InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, + seg_size, false); thread_args[i].private_data.thread_num = i + 1; thread_args[i].startpoint = startpoint; @@ -362,7 +372,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, /* Adjust startpoint to the next thread */ if (nextSegNoToRead == 0) - XLByteToSeg(startpoint, nextSegNoToRead); + GetXLogSegNo(startpoint, nextSegNoToRead, seg_size); nextSegNoToRead++; /* @@ -371,7 +381,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, */ if (nextSegNoToRead > endSegNo) break; - XLogSegNoOffsetToRecPtr(nextSegNoToRead, 0, startpoint); + GetXLogRecPtr(nextSegNoToRead, 0, seg_size, startpoint); } /* Run threads */ @@ -405,7 +415,8 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, */ static void validate_backup_wal_from_start_to_stop(pgBackup *backup, - char *backup_xlog_path, TimeLineID tli) + char *backup_xlog_path, TimeLineID tli, + uint32 xlog_seg_size) { XLogRecPtr startpoint = backup->start_lsn; XLogRecord *record; @@ -414,7 +425,8 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, XLogPageReadPrivate private; bool got_endpoint = false; - xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, true); + xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, + xlog_seg_size, true); while (true) { @@ -468,12 +480,10 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, * up to the given recovery target. */ void -validate_wal(pgBackup *backup, - const char *archivedir, - time_t target_time, - TransactionId target_xid, +validate_wal(pgBackup *backup, const char *archivedir, + time_t target_time, TransactionId target_xid, XLogRecPtr target_lsn, - TimeLineID tli) + TimeLineID tli, uint32 seg_size) { XLogRecPtr startpoint = backup->start_lsn; const char *backup_id; @@ -510,10 +520,12 @@ validate_wal(pgBackup *backup, snprintf(backup_xlog_path, sizeof(backup_xlog_path), "/%s/%s/%s/%s", backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); - validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli); + validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, + seg_size); } else - validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli); + validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli, + seg_size); if (backup->status == BACKUP_STATUS_CORRUPT) { @@ -543,7 +555,8 @@ validate_wal(pgBackup *backup, * up to the given recovery target. * In any case we cannot restore to the point before stop_lsn. */ - xlogreader = InitXLogPageRead(&private, archivedir, tli, true); + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, + true); /* We can restore at least up to the backup end */ time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); @@ -639,7 +652,7 @@ validate_wal(pgBackup *backup, * pg_stop_backup(). */ bool -read_recovery_info(const char *archivedir, TimeLineID tli, +read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, time_t *recovery_time, TransactionId *recovery_xid) { @@ -656,7 +669,7 @@ read_recovery_info(const char *archivedir, TimeLineID tli, elog(ERROR, "Invalid stop_lsn value %X/%X", (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - xlogreader = InitXLogPageRead(&private, archivedir, tli, true); + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); /* Read records from stop_lsn down to start_lsn */ do @@ -711,7 +724,7 @@ cleanup: */ bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, - TimeLineID target_tli) + TimeLineID target_tli, uint32 seg_size) { XLogReaderState *xlogreader; XLogPageReadPrivate private; @@ -722,7 +735,8 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, elog(ERROR, "Invalid target_lsn value %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); - xlogreader = InitXLogPageRead(&private, archivedir, target_tli, true); + xlogreader = InitXLogPageRead(&private, archivedir, target_tli, seg_size, + true); res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; /* Didn't find 'target_lsn' and there is no error, return false */ @@ -761,13 +775,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, uint32 targetPageOff; private_data = (XLogPageReadPrivate *) xlogreader->private_data; - targetPageOff = targetPagePtr % XLogSegSize; + targetPageOff = targetPagePtr % private_data->xlog_seg_size; /* * See if we need to switch to a new segment because the requested record * is not in the currently open one. */ - if (!XLByteInSeg(targetPagePtr, private_data->xlogsegno)) + if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size)) { elog(VERBOSE, "Thread [%d]: Need to switch to segno next to %X/%X, current LSN %X/%X", private_data->thread_num, @@ -805,22 +820,24 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, } } - XLByteToSeg(targetPagePtr, private_data->xlogsegno); + GetXLogSegNo(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size); /* Try to switch to the next WAL segment */ if (!private_data->xlogexists) { char xlogfname[MAXFNAMELEN]; - XLogFileName(xlogfname, private_data->tli, private_data->xlogsegno); + GetXLogFileName(xlogfname, private_data->tli, private_data->xlogsegno, + private_data->xlog_seg_size); snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", private_data->archivedir, xlogfname); if (fileExists(private_data->xlogpath)) { elog(LOG, "Thread [%d]: Opening WAL segment \"%s\"", - private_data->thread_num, - private_data->xlogpath); + private_data->thread_num, + private_data->xlogpath); private_data->xlogexists = true; private_data->xlogfile = open(private_data->xlogpath, @@ -829,9 +846,9 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (private_data->xlogfile < 0) { elog(WARNING, "Thread [%d]: Could not open WAL segment \"%s\": %s", - private_data->thread_num, - private_data->xlogpath, - strerror(errno)); + private_data->thread_num, + private_data->xlogpath, + strerror(errno)); return -1; } } @@ -876,14 +893,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) { elog(WARNING, "Thread [%d]: Could not seek in WAL segment \"%s\": %s", - private_data->thread_num, private_data->xlogpath, strerror(errno)); + private_data->thread_num, private_data->xlogpath, strerror(errno)); return -1; } if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Thread [%d]: Could not read from WAL segment \"%s\": %s", - private_data->thread_num, private_data->xlogpath, strerror(errno)); + private_data->thread_num, private_data->xlogpath, strerror(errno)); return -1; } } @@ -919,18 +936,24 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, */ static XLogReaderState * InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, - TimeLineID tli, bool allocate_reader) + TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader) { XLogReaderState *xlogreader = NULL; MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); private_data->archivedir = archivedir; private_data->tli = tli; + private_data->xlog_seg_size = xlog_seg_size; private_data->xlogfile = -1; if (allocate_reader) { +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif if (xlogreader == NULL) elog(ERROR, "out of memory"); xlogreader->system_identifier = system_identifier; @@ -974,8 +997,8 @@ PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) */ if (!private_data->xlogexists) elog(elevel, "Thread [%d]: WAL segment \"%s\" is absent", - private_data->thread_num, - private_data->xlogpath); + private_data->thread_num, + private_data->xlogpath); else if (private_data->xlogfile != -1) elog(elevel, "Thread [%d]: Possible WAL corruption. " "Error has occured during reading WAL segment \"%s\"", diff --git a/src/pg_probackup.c b/src/pg_probackup.c index bcee49e3..35a9ea83 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -93,6 +93,16 @@ bool compress_shortcut = false; char *instance_name; uint64 system_identifier = 0; +/* + * Starting from PostgreSQL 11 WAL segment size may vary. Prior to + * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. + */ +#if PG_VERSION_NUM >= 110000 +uint32 xlog_seg_size = 0; +#else +uint32 xlog_seg_size = XLOG_SEG_SIZE; +#endif + /* archive push options */ static char *wal_file_path; static char *wal_file_name; @@ -184,6 +194,9 @@ static pgut_option options[] = /* other options */ { 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, { 's', 151, "instance", &instance_name, SOURCE_CMDLINE }, +#if PG_VERSION_NUM >= 110000 + { 'u', 152, "xlog-seg-size", &xlog_seg_size, SOURCE_FILE_STRICT}, +#endif /* archive-push options */ { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, @@ -211,6 +224,14 @@ main(int argc, char *argv[]) PROGRAM_NAME = get_progname(argv[0]); set_pglocale_pgservice(argv[0], "pgscripts"); +#if PG_VERSION_NUM >= 110000 + /* + * Reset WAL segment size, we will retreive it using RetrieveWalSegSize() + * later. + */ + WalSegSz = 0; +#endif + /* * Save main thread's tid. It is used call exit() in case of errors. */ @@ -393,7 +414,7 @@ main(int argc, char *argv[]) /* Read options from configuration file */ join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - pgut_readopt(path, options, ERROR); + pgut_readopt(path, options, ERROR, true); } /* Initialize logger */ @@ -406,6 +427,14 @@ main(int argc, char *argv[]) if (pgdata != NULL && !is_absolute_path(pgdata)) elog(ERROR, "-D, --pgdata must be an absolute path"); +#if PG_VERSION_NUM >= 110000 + /* Check xlog-seg-size option */ + if (instance_name && + backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != ADD_INSTANCE_CMD && !IsValidWalSegSize(xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", xlog_seg_size); +#endif + /* Sanity check of --backup-id option */ if (backup_id_string != NULL) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0301da83..ed7defc2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -172,11 +172,13 @@ typedef enum ShowFormat typedef struct pgBackupConfig { uint64 system_identifier; - char *pgdata; - const char *pgdatabase; - const char *pghost; - const char *pgport; - const char *pguser; + uint32 xlog_seg_size; + + char *pgdata; + const char *pgdatabase; + const char *pghost; + const char *pgport; + const char *pguser; const char *master_host; const char *master_port; @@ -324,6 +326,26 @@ typedef struct strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) +#if PG_VERSION_NUM >= 110000 +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) +#else +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo) +#endif + /* directory options */ extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; @@ -384,6 +406,7 @@ extern const char* deparse_compress_alg(int alg); /* other options */ extern char *instance_name; extern uint64 system_identifier; +extern uint32 xlog_seg_size; /* show options */ extern ShowFormat show_format; @@ -441,6 +464,8 @@ extern void writeBackupCatalogConfig(FILE *out, pgBackupConfig *config); extern void writeBackupCatalogConfigFile(pgBackupConfig *config); extern pgBackupConfig* readBackupCatalogConfigFile(void); +extern uint32 get_config_xlog_seg_size(void); + /* in show.c */ extern int do_show(time_t requested_backup_id); @@ -540,23 +565,23 @@ extern void get_wal_file(const char *from_path, const char *to_path); extern bool calc_file_checksum(pgFile *file); /* parsexlog.c */ -extern void extractPageMap(const char *datadir, - XLogRecPtr startpoint, - TimeLineID tli, - XLogRecPtr endpoint, - parray *backup_files_list); +extern void extractPageMap(const char *archivedir, + TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, + parray *files); extern void validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, XLogRecPtr target_lsn, - TimeLineID tli); + TimeLineID tli, uint32 seg_size); extern bool read_recovery_info(const char *archivedir, TimeLineID tli, + uint32 seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, time_t *recovery_time, TransactionId *recovery_xid); extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, - TimeLineID target_tli); + TimeLineID target_tli, uint32 seg_size); /* in util.c */ extern TimeLineID get_current_timeline(bool safe); @@ -564,6 +589,7 @@ extern XLogRecPtr get_checkpoint_location(PGconn *conn); extern uint64 get_system_identifier(char *pgdata); extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); +extern uint32 get_xlog_seg_size(char *pgdata_path); extern void sanityChecks(void); extern void time2iso(char *buf, size_t len, time_t time); diff --git a/src/restore.c b/src/restore.c index 6c87f59b..77fcbaa4 100644 --- a/src/restore.c +++ b/src/restore.c @@ -318,8 +318,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * because it's needed to form the name of xlog file. */ validate_wal(dest_backup, arclog_path, rt->recovery_target_time, - rt->recovery_target_xid, rt->recovery_target_lsn, - base_full_backup->tli); + rt->recovery_target_xid, rt->recovery_target_lsn, + base_full_backup->tli, xlog_seg_size); } /* Orphinize every OK descendant of corrupted backup */ else diff --git a/src/util.c b/src/util.c index 788df42c..aed34ce8 100644 --- a/src/util.c +++ b/src/util.c @@ -13,6 +13,9 @@ #include #include "storage/bufpage.h" +#if PG_VERSION_NUM >= 110000 +#include "streamutil.h" +#endif const char * base36enc(long unsigned int value) @@ -212,6 +215,27 @@ get_remote_system_identifier(PGconn *conn) #endif } +uint32 +get_xlog_seg_size(char *pgdata_path) +{ +#if PG_VERSION_NUM >= 110000 + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.xlog_seg_size; +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + uint32 get_data_checksum_version(bool safe) { diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 3dc7d57d..4e181d1a 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -91,11 +91,6 @@ static const unit_conversion memory_unit_conversion_table[] = {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, - {"TB", OPTION_UNIT_XSEGS, (1024 * 1024 * 1024) / (XLOG_SEG_SIZE / 1024)}, - {"GB", OPTION_UNIT_XSEGS, (1024 * 1024) / (XLOG_SEG_SIZE / 1024)}, - {"MB", OPTION_UNIT_XSEGS, -(XLOG_SEG_SIZE / (1024 * 1024))}, - {"kB", OPTION_UNIT_XSEGS, -(XLOG_SEG_SIZE / 1024)}, - {""} /* end of table marker */ }; @@ -1140,7 +1135,7 @@ key_equals(const char *lhs, const char *rhs) * Return number of parsed options */ int -pgut_readopt(const char *path, pgut_option options[], int elevel) +pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict) { FILE *fp; char buf[1024]; @@ -1180,7 +1175,7 @@ pgut_readopt(const char *path, pgut_option options[], int elevel) break; } } - if (!options[i].type) + if (strict && !options[i].type) elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path); } } diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 0947fb7f..fedb99b0 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -111,7 +111,8 @@ extern bool in_cleanup; extern bool in_password; /* User prompts password */ extern int pgut_getopt(int argc, char **argv, pgut_option options[]); -extern int pgut_readopt(const char *path, pgut_option options[], int elevel); +extern int pgut_readopt(const char *path, pgut_option options[], int elevel, + bool strict); extern void pgut_getopt_env(pgut_option options[]); extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); diff --git a/src/validate.c b/src/validate.c index 7b7771ac..e7893ae8 100644 --- a/src/validate.c +++ b/src/validate.c @@ -259,6 +259,8 @@ do_validate_all(void) instance_name = dent->d_name; sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + xlog_seg_size = get_config_xlog_seg_size(); + do_validate_instance(); } } @@ -381,7 +383,7 @@ do_validate_instance(void) /* Validate corresponding WAL files */ if (current_backup->status == BACKUP_STATUS_OK) validate_wal(current_backup, arclog_path, 0, - 0, 0, base_full_backup->tli); + 0, 0, base_full_backup->tli, xlog_seg_size); /* * Mark every descendant of corrupted backup as orphan @@ -468,7 +470,8 @@ do_validate_instance(void) //tmp_backup = find_parent_full_backup(dest_backup); /* Revalidation successful, validate corresponding WAL files */ validate_wal(backup, arclog_path, 0, - 0, 0, current_backup->tli); + 0, 0, current_backup->tli, + xlog_seg_size); } }