1
0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-24 08:52:38 +02:00

Port to PostgreSQL 11

This commit is contained in:
Arthur Zakirov 2018-10-09 12:51:41 +03:00
parent cfb63fb2aa
commit 78adfdfa3b
15 changed files with 264 additions and 96 deletions

View File

@ -32,7 +32,7 @@ else
srchome=$(top_srcdir)
endif
ifeq ($(MAJORVERSION),10)
ifneq (,$(filter 10 11 12,$(MAJORVERSION)))
OBJS += src/walmethods.o
EXTRA_CLEAN += src/walmethods.c src/walmethods.h
INCLUDES += src/walmethods.h
@ -64,7 +64,7 @@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@
ifeq ($(MAJORVERSION),10)
ifneq (,$(filter 10 11 12,$(MAJORVERSION)))
src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@
src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h

View File

@ -650,8 +650,9 @@ do_backup_instance(void)
* reading WAL segments present in archives up to the point
* where this backup has started.
*/
extractPageMap(arclog_path, prev_backup->start_lsn, current.tli,
current.start_lsn, backup_files_list);
extractPageMap(arclog_path, current.tli, xlog_seg_size,
prev_backup->start_lsn, current.start_lsn,
backup_files_list);
}
else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
{
@ -827,6 +828,11 @@ do_backup(time_t start_time)
current.primary_conninfo = pgut_get_conninfo_string(backup_conn);
#if PG_VERSION_NUM >= 110000
if (!RetrieveWalSegSize(backup_conn))
elog(ERROR, "Failed to retreive wal_segment_size");
#endif
current.compress_alg = compress_alg;
current.compress_level = compress_level;
@ -918,8 +924,9 @@ do_backup(time_t start_time)
/* compute size of wal files of this backup stored in the archive */
if (!current.stream)
{
current.wal_bytes = XLOG_SEG_SIZE *
(current.stop_lsn/XLogSegSize - current.start_lsn/XLogSegSize + 1);
current.wal_bytes = xlog_seg_size *
(current.stop_lsn / xlog_seg_size -
current.start_lsn / xlog_seg_size + 1);
}
/* Backup is done. Update backup status */
@ -1467,10 +1474,10 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment)
tli = get_current_timeline(false);
/* Compute the name of the WAL file containig requested LSN */
XLByteToSeg(lsn, targetSegNo);
GetXLogSegNo(lsn, targetSegNo, xlog_seg_size);
if (wait_prev_segment)
targetSegNo--;
XLogFileName(wal_segment, tli, targetSegNo);
GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size);
/*
* In pg_start_backup we wait for 'lsn' in 'pg_wal' directory iff it is
@ -1536,7 +1543,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment)
/*
* A WAL segment found. Check LSN on it.
*/
if (wal_contains_lsn(wal_segment_dir, lsn, tli))
if (wal_contains_lsn(wal_segment_dir, lsn, tli, xlog_seg_size))
/* Target LSN was found */
{
elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn);
@ -1946,7 +1953,7 @@ pg_stop_backup(pgBackup *backup)
elog(LOG, "Getting the Recovery Time from WAL");
if (!read_recovery_info(xlog_path, backup->tli,
if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size,
backup->start_lsn, backup->stop_lsn,
&backup->recovery_time, &backup->recovery_xid))
{
@ -2553,7 +2560,7 @@ StreamLog(void *arg)
/*
* Always start streaming at the beginning of a segment
*/
startpos -= startpos % XLOG_SEG_SIZE;
startpos -= startpos % xlog_seg_size;
/* Initialize timeout */
stream_stop_timeout = 0;

View File

@ -572,23 +572,23 @@ readBackupControlFile(const char *path)
pgBackupInit(backup);
if (access(path, F_OK) != 0)
{
elog(WARNING, "control file \"%s\" doesn't exist", path);
elog(WARNING, "Control file \"%s\" doesn't exist", path);
pgBackupFree(backup);
return NULL;
}
parsed_options = pgut_readopt(path, options, WARNING);
parsed_options = pgut_readopt(path, options, WARNING, true);
if (parsed_options == 0)
{
elog(WARNING, "control file \"%s\" is empty", path);
elog(WARNING, "Control file \"%s\" is empty", path);
pgBackupFree(backup);
return NULL;
}
if (backup->start_time == 0)
{
elog(WARNING, "invalid ID/start-time, control file \"%s\" is corrupted", path);
elog(WARNING, "Invalid ID/start-time, control file \"%s\" is corrupted", path);
pgBackupFree(backup);
return NULL;
}
@ -607,7 +607,7 @@ readBackupControlFile(const char *path)
if (sscanf(start_lsn, "%X/%X", &xlogid, &xrecoff) == 2)
backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff;
else
elog(WARNING, "invalid START_LSN \"%s\"", start_lsn);
elog(WARNING, "Invalid START_LSN \"%s\"", start_lsn);
free(start_lsn);
}
@ -619,7 +619,7 @@ readBackupControlFile(const char *path)
if (sscanf(stop_lsn, "%X/%X", &xlogid, &xrecoff) == 2)
backup->stop_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff;
else
elog(WARNING, "invalid STOP_LSN \"%s\"", stop_lsn);
elog(WARNING, "Invalid STOP_LSN \"%s\"", stop_lsn);
free(stop_lsn);
}
@ -644,7 +644,7 @@ readBackupControlFile(const char *path)
else if (strcmp(status, "CORRUPT") == 0)
backup->status = BACKUP_STATUS_CORRUPT;
else
elog(WARNING, "invalid STATUS \"%s\"", status);
elog(WARNING, "Invalid STATUS \"%s\"", status);
free(status);
}

View File

@ -102,6 +102,13 @@ void
pgBackupConfigInit(pgBackupConfig *config)
{
config->system_identifier = 0;
#if PG_VERSION_NUM >= 110000
config->xlog_seg_size = 0;
#else
config->xlog_seg_size = XLOG_SEG_SIZE;
#endif
config->pgdata = NULL;
config->pgdatabase = NULL;
config->pghost = NULL;
@ -140,6 +147,9 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config)
fprintf(out, "#Backup instance info\n");
fprintf(out, "PGDATA = %s\n", config->pgdata);
fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier);
#if PG_VERSION_NUM >= 110000
fprintf(out, "xlog-seg-size = %u\n", config->xlog_seg_size);
#endif
fprintf(out, "#Connection parameters:\n");
if (config->pgdatabase)
@ -253,6 +263,9 @@ readBackupCatalogConfigFile(void)
{ 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS },
/* other options */
{ 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT },
#if PG_VERSION_NUM >= 110000
{'u', 0, "xlog-seg-size", &config->xlog_seg_size, SOURCE_FILE_STRICT},
#endif
/* archive options */
{ 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS },
{0}
@ -263,11 +276,44 @@ readBackupCatalogConfigFile(void)
join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
pgBackupConfigInit(config);
pgut_readopt(path, options, ERROR);
pgut_readopt(path, options, ERROR, true);
#if PG_VERSION_NUM >= 110000
if (!IsValidWalSegSize(config->xlog_seg_size))
elog(ERROR, "Invalid WAL segment size %u", config->xlog_seg_size);
#endif
return config;
}
/*
* Read xlog-seg-size from BACKUP_CATALOG_CONF_FILE.
*/
uint32
get_config_xlog_seg_size(void)
{
#if PG_VERSION_NUM >= 110000
char path[MAXPGPATH];
uint32 seg_size;
pgut_option options[] =
{
{'u', 0, "xlog-seg-size", &seg_size, SOURCE_FILE_STRICT},
{0}
};
join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
pgut_readopt(path, options, ERROR, false);
if (!IsValidWalSegSize(seg_size))
elog(ERROR, "Invalid WAL segment size %u", seg_size);
return seg_size;
#else
return (uint32) XLOG_SEG_SIZE;
#endif
}
static void
opt_log_level_console(pgut_option *opt, const char *arg)
{
@ -349,6 +395,11 @@ show_configure_json(pgBackupConfig *config)
json_add_key(buf, "system-identifier", json_level, true);
appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier);
#if PG_VERSION_NUM >= 110000
json_add_key(buf, "xlog-seg-size", json_level, true);
appendPQExpBuffer(buf, "%u", config->xlog_seg_size);
#endif
/* Connection parameters */
if (config->pgdatabase)
json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true);

View File

@ -15,7 +15,8 @@
#include <unistd.h>
static int pgBackupDeleteFiles(pgBackup *backup);
static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli);
static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
uint32 xlog_seg_size);
int
do_delete(time_t backup_id)
@ -23,8 +24,8 @@ do_delete(time_t backup_id)
int i;
parray *backup_list,
*delete_list;
pgBackup *target_backup = NULL;
time_t parent_id = 0;
bool backup_found = false;
XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
TimeLineID oldest_tli = 0;
@ -56,9 +57,9 @@ do_delete(time_t backup_id)
/* Save backup id to retreive increment backups */
parent_id = backup->start_time;
backup_found = true;
target_backup = backup;
}
else if (backup_found)
else if (target_backup)
{
if (backup->backup_mode != BACKUP_MODE_FULL &&
backup->parent_backup == parent_id)
@ -93,6 +94,8 @@ do_delete(time_t backup_id)
/* Clean WAL segments */
if (delete_wal)
{
Assert(target_backup);
/* Find oldest LSN, used by backups */
for (i = (int) parray_num(backup_list) - 1; i >= 0; i--)
{
@ -106,7 +109,7 @@ do_delete(time_t backup_id)
}
}
delete_walfiles(oldest_lsn, oldest_tli);
delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size);
}
/* cleanup */
@ -225,7 +228,7 @@ do_retention_purge(void)
/* Purge WAL files */
if (delete_wal)
{
delete_walfiles(oldest_lsn, oldest_tli);
delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size);
}
/* Cleanup */
@ -313,7 +316,8 @@ pgBackupDeleteFiles(pgBackup *backup)
* oldest_lsn.
*/
static void
delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli)
delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
uint32 xlog_seg_size)
{
XLogSegNo targetSegNo;
char oldestSegmentNeeded[MAXFNAMELEN];
@ -329,8 +333,9 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli)
if (!XLogRecPtrIsInvalid(oldest_lsn))
{
XLByteToSeg(oldest_lsn, targetSegNo);
XLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo);
GetXLogSegNo(oldest_lsn, targetSegNo, xlog_seg_size);
GetXLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo,
xlog_seg_size);
elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded);
}
@ -436,7 +441,7 @@ do_delete_instance(void)
parray_free(backup_list);
/* Delete all wal files. */
delete_walfiles(InvalidXLogRecPtr, 0);
delete_walfiles(InvalidXLogRecPtr, 0, xlog_seg_size);
/* Delete backup instance config file */
join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);

View File

@ -54,7 +54,7 @@ do_add_instance(void)
{
char path[MAXPGPATH];
char arclog_path_dir[MAXPGPATH];
struct stat st;
struct stat st;
pgBackupConfig *config = pgut_new(pgBackupConfig);
/* PGDATA is always required */
@ -64,6 +64,8 @@ do_add_instance(void)
/* Read system_identifier from PGDATA */
system_identifier = get_system_identifier(pgdata);
/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
xlog_seg_size = get_xlog_seg_size(pgdata);
/* Ensure that all root directories already exist */
if (access(backup_path, F_OK) != 0)
@ -97,6 +99,7 @@ do_add_instance(void)
*/
pgBackupConfigInit(config);
config->system_identifier = system_identifier;
config->xlog_seg_size = xlog_seg_size;
config->pgdata = pgdata;
writeBackupCatalogConfigFile(config);

View File

@ -320,9 +320,10 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
to_backup->data_bytes += file->write_size;
}
/* compute size of wal files of this backup stored in the archive */
if (!current.stream)
to_backup->wal_bytes = XLOG_SEG_SIZE *
(to_backup->stop_lsn / XLogSegSize - to_backup->start_lsn / XLogSegSize + 1);
if (!to_backup->stream)
to_backup->wal_bytes = xlog_seg_size *
(to_backup->stop_lsn / xlog_seg_size -
to_backup->start_lsn / xlog_seg_size + 1);
else
to_backup->wal_bytes = BYTES_INVALID;

View File

@ -89,6 +89,7 @@ typedef struct XLogPageReadPrivate
int thread_num;
const char *archivedir;
TimeLineID tli;
uint32 xlog_seg_size;
bool manual_switch;
bool need_switch;
@ -126,7 +127,8 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
TimeLineID *pageTLI);
static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data,
const char *archivedir,
TimeLineID tli, bool allocate_reader);
TimeLineID tli, uint32 xlog_seg_size,
bool allocate_reader);
static void CleanupXLogPageRead(XLogReaderState *xlogreader);
static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data,
int elevel);
@ -160,7 +162,8 @@ switchToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg)
return false;
/* Adjust next record position */
XLogSegNoOffsetToRecPtr(private_data->xlogsegno, 0, arg->startpoint);
GetXLogRecPtr(private_data->xlogsegno, 0,
private_data->xlog_seg_size, arg->startpoint);
/* We need to close previously opened file if it wasn't closed earlier */
CleanupXLogPageRead(xlogreader);
/* Skip over the page header and contrecord if any */
@ -200,7 +203,12 @@ doExtractPageMap(void *arg)
char *errormsg;
private_data = &extract_arg->private_data;
#if PG_VERSION_NUM >= 110000
xlogreader = XLogReaderAllocate(private_data->xlog_seg_size,
&SimpleXLogPageRead, private_data);
#else
xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data);
#endif
if (xlogreader == NULL)
elog(ERROR, "Thread [%d]: out of memory", private_data->thread_num);
xlogreader->system_identifier = system_identifier;
@ -235,7 +243,7 @@ doExtractPageMap(void *arg)
if (interrupted)
elog(ERROR, "Thread [%d]: Interrupted during WAL reading",
private_data->thread_num);
private_data->thread_num);
/*
* We need to switch to the next WAL segment after reading previous
@ -292,7 +300,8 @@ doExtractPageMap(void *arg)
/* continue reading at next record */
extract_arg->startpoint = InvalidXLogRecPtr;
XLByteToSeg(xlogreader->EndRecPtr, nextSegNo);
GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo,
private_data->xlog_seg_size);
} while (nextSegNo <= extract_arg->endSegNo &&
xlogreader->ReadRecPtr < extract_arg->endpoint);
@ -312,8 +321,8 @@ doExtractPageMap(void *arg)
* file.
*/
void
extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
XLogRecPtr endpoint, parray *files)
extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size,
XLogRecPtr startpoint, XLogRecPtr endpoint, parray *files)
{
int i;
int threads_need = 0;
@ -333,7 +342,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
elog(ERROR, "Invalid endpoint value %X/%X",
(uint32) (endpoint >> 32), (uint32) (endpoint));
XLByteToSeg(endpoint, endSegNo);
GetXLogSegNo(endpoint, endSegNo, seg_size);
nextSegNoToRead = 0;
time(&start_time);
@ -349,7 +358,8 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
*/
for (i = 0; i < num_threads; i++)
{
InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, false);
InitXLogPageRead(&thread_args[i].private_data, archivedir, tli,
seg_size, false);
thread_args[i].private_data.thread_num = i + 1;
thread_args[i].startpoint = startpoint;
@ -362,7 +372,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
/* Adjust startpoint to the next thread */
if (nextSegNoToRead == 0)
XLByteToSeg(startpoint, nextSegNoToRead);
GetXLogSegNo(startpoint, nextSegNoToRead, seg_size);
nextSegNoToRead++;
/*
@ -371,7 +381,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
*/
if (nextSegNoToRead > endSegNo)
break;
XLogSegNoOffsetToRecPtr(nextSegNoToRead, 0, startpoint);
GetXLogRecPtr(nextSegNoToRead, 0, seg_size, startpoint);
}
/* Run threads */
@ -405,7 +415,8 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
*/
static void
validate_backup_wal_from_start_to_stop(pgBackup *backup,
char *backup_xlog_path, TimeLineID tli)
char *backup_xlog_path, TimeLineID tli,
uint32 xlog_seg_size)
{
XLogRecPtr startpoint = backup->start_lsn;
XLogRecord *record;
@ -414,7 +425,8 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup,
XLogPageReadPrivate private;
bool got_endpoint = false;
xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, true);
xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli,
xlog_seg_size, true);
while (true)
{
@ -468,12 +480,10 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup,
* up to the given recovery target.
*/
void
validate_wal(pgBackup *backup,
const char *archivedir,
time_t target_time,
TransactionId target_xid,
validate_wal(pgBackup *backup, const char *archivedir,
time_t target_time, TransactionId target_xid,
XLogRecPtr target_lsn,
TimeLineID tli)
TimeLineID tli, uint32 seg_size)
{
XLogRecPtr startpoint = backup->start_lsn;
const char *backup_id;
@ -510,10 +520,12 @@ validate_wal(pgBackup *backup,
snprintf(backup_xlog_path, sizeof(backup_xlog_path), "/%s/%s/%s/%s",
backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR);
validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli);
validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli,
seg_size);
}
else
validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli);
validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli,
seg_size);
if (backup->status == BACKUP_STATUS_CORRUPT)
{
@ -543,7 +555,8 @@ validate_wal(pgBackup *backup,
* up to the given recovery target.
* In any case we cannot restore to the point before stop_lsn.
*/
xlogreader = InitXLogPageRead(&private, archivedir, tli, true);
xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size,
true);
/* We can restore at least up to the backup end */
time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time);
@ -639,7 +652,7 @@ validate_wal(pgBackup *backup,
* pg_stop_backup().
*/
bool
read_recovery_info(const char *archivedir, TimeLineID tli,
read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size,
XLogRecPtr start_lsn, XLogRecPtr stop_lsn,
time_t *recovery_time, TransactionId *recovery_xid)
{
@ -656,7 +669,7 @@ read_recovery_info(const char *archivedir, TimeLineID tli,
elog(ERROR, "Invalid stop_lsn value %X/%X",
(uint32) (stop_lsn >> 32), (uint32) (stop_lsn));
xlogreader = InitXLogPageRead(&private, archivedir, tli, true);
xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true);
/* Read records from stop_lsn down to start_lsn */
do
@ -711,7 +724,7 @@ cleanup:
*/
bool
wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn,
TimeLineID target_tli)
TimeLineID target_tli, uint32 seg_size)
{
XLogReaderState *xlogreader;
XLogPageReadPrivate private;
@ -722,7 +735,8 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn,
elog(ERROR, "Invalid target_lsn value %X/%X",
(uint32) (target_lsn >> 32), (uint32) (target_lsn));
xlogreader = InitXLogPageRead(&private, archivedir, target_tli, true);
xlogreader = InitXLogPageRead(&private, archivedir, target_tli, seg_size,
true);
res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL;
/* Didn't find 'target_lsn' and there is no error, return false */
@ -761,13 +775,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
uint32 targetPageOff;
private_data = (XLogPageReadPrivate *) xlogreader->private_data;
targetPageOff = targetPagePtr % XLogSegSize;
targetPageOff = targetPagePtr % private_data->xlog_seg_size;
/*
* See if we need to switch to a new segment because the requested record
* is not in the currently open one.
*/
if (!XLByteInSeg(targetPagePtr, private_data->xlogsegno))
if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno,
private_data->xlog_seg_size))
{
elog(VERBOSE, "Thread [%d]: Need to switch to segno next to %X/%X, current LSN %X/%X",
private_data->thread_num,
@ -805,22 +820,24 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
}
}
XLByteToSeg(targetPagePtr, private_data->xlogsegno);
GetXLogSegNo(targetPagePtr, private_data->xlogsegno,
private_data->xlog_seg_size);
/* Try to switch to the next WAL segment */
if (!private_data->xlogexists)
{
char xlogfname[MAXFNAMELEN];
XLogFileName(xlogfname, private_data->tli, private_data->xlogsegno);
GetXLogFileName(xlogfname, private_data->tli, private_data->xlogsegno,
private_data->xlog_seg_size);
snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s",
private_data->archivedir, xlogfname);
if (fileExists(private_data->xlogpath))
{
elog(LOG, "Thread [%d]: Opening WAL segment \"%s\"",
private_data->thread_num,
private_data->xlogpath);
private_data->thread_num,
private_data->xlogpath);
private_data->xlogexists = true;
private_data->xlogfile = open(private_data->xlogpath,
@ -829,9 +846,9 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private_data->xlogfile < 0)
{
elog(WARNING, "Thread [%d]: Could not open WAL segment \"%s\": %s",
private_data->thread_num,
private_data->xlogpath,
strerror(errno));
private_data->thread_num,
private_data->xlogpath,
strerror(errno));
return -1;
}
}
@ -876,14 +893,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0)
{
elog(WARNING, "Thread [%d]: Could not seek in WAL segment \"%s\": %s",
private_data->thread_num, private_data->xlogpath, strerror(errno));
private_data->thread_num, private_data->xlogpath, strerror(errno));
return -1;
}
if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ)
{
elog(WARNING, "Thread [%d]: Could not read from WAL segment \"%s\": %s",
private_data->thread_num, private_data->xlogpath, strerror(errno));
private_data->thread_num, private_data->xlogpath, strerror(errno));
return -1;
}
}
@ -919,18 +936,24 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
*/
static XLogReaderState *
InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir,
TimeLineID tli, bool allocate_reader)
TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader)
{
XLogReaderState *xlogreader = NULL;
MemSet(private_data, 0, sizeof(XLogPageReadPrivate));
private_data->archivedir = archivedir;
private_data->tli = tli;
private_data->xlog_seg_size = xlog_seg_size;
private_data->xlogfile = -1;
if (allocate_reader)
{
#if PG_VERSION_NUM >= 110000
xlogreader = XLogReaderAllocate(xlog_seg_size,
&SimpleXLogPageRead, private_data);
#else
xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data);
#endif
if (xlogreader == NULL)
elog(ERROR, "out of memory");
xlogreader->system_identifier = system_identifier;
@ -974,8 +997,8 @@ PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel)
*/
if (!private_data->xlogexists)
elog(elevel, "Thread [%d]: WAL segment \"%s\" is absent",
private_data->thread_num,
private_data->xlogpath);
private_data->thread_num,
private_data->xlogpath);
else if (private_data->xlogfile != -1)
elog(elevel, "Thread [%d]: Possible WAL corruption. "
"Error has occured during reading WAL segment \"%s\"",

View File

@ -93,6 +93,16 @@ bool compress_shortcut = false;
char *instance_name;
uint64 system_identifier = 0;
/*
* Starting from PostgreSQL 11 WAL segment size may vary. Prior to
* PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE.
*/
#if PG_VERSION_NUM >= 110000
uint32 xlog_seg_size = 0;
#else
uint32 xlog_seg_size = XLOG_SEG_SIZE;
#endif
/* archive push options */
static char *wal_file_path;
static char *wal_file_name;
@ -184,6 +194,9 @@ static pgut_option options[] =
/* other options */
{ 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT },
{ 's', 151, "instance", &instance_name, SOURCE_CMDLINE },
#if PG_VERSION_NUM >= 110000
{ 'u', 152, "xlog-seg-size", &xlog_seg_size, SOURCE_FILE_STRICT},
#endif
/* archive-push options */
{ 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE },
{ 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE },
@ -211,6 +224,14 @@ main(int argc, char *argv[])
PROGRAM_NAME = get_progname(argv[0]);
set_pglocale_pgservice(argv[0], "pgscripts");
#if PG_VERSION_NUM >= 110000
/*
* Reset WAL segment size, we will retreive it using RetrieveWalSegSize()
* later.
*/
WalSegSz = 0;
#endif
/*
* Save main thread's tid. It is used call exit() in case of errors.
*/
@ -393,7 +414,7 @@ main(int argc, char *argv[])
/* Read options from configuration file */
join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
pgut_readopt(path, options, ERROR);
pgut_readopt(path, options, ERROR, true);
}
/* Initialize logger */
@ -406,6 +427,14 @@ main(int argc, char *argv[])
if (pgdata != NULL && !is_absolute_path(pgdata))
elog(ERROR, "-D, --pgdata must be an absolute path");
#if PG_VERSION_NUM >= 110000
/* Check xlog-seg-size option */
if (instance_name &&
backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD &&
backup_subcmd != ADD_INSTANCE_CMD && !IsValidWalSegSize(xlog_seg_size))
elog(ERROR, "Invalid WAL segment size %u", xlog_seg_size);
#endif
/* Sanity check of --backup-id option */
if (backup_id_string != NULL)
{

View File

@ -172,11 +172,13 @@ typedef enum ShowFormat
typedef struct pgBackupConfig
{
uint64 system_identifier;
char *pgdata;
const char *pgdatabase;
const char *pghost;
const char *pgport;
const char *pguser;
uint32 xlog_seg_size;
char *pgdata;
const char *pgdatabase;
const char *pghost;
const char *pgport;
const char *pguser;
const char *master_host;
const char *master_port;
@ -324,6 +326,26 @@ typedef struct
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0)
#if PG_VERSION_NUM >= 110000
#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \
XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \
XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \
XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \
XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes)
#else
#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \
XLByteToSeg(xlrp, logSegNo)
#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \
XLogSegNoOffsetToRecPtr(segno, offset, dest)
#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \
XLogFileName(fname, tli, logSegNo)
#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \
XLByteInSeg(xlrp, logSegNo)
#endif
/* directory options */
extern char *backup_path;
extern char backup_instance_path[MAXPGPATH];
@ -384,6 +406,7 @@ extern const char* deparse_compress_alg(int alg);
/* other options */
extern char *instance_name;
extern uint64 system_identifier;
extern uint32 xlog_seg_size;
/* show options */
extern ShowFormat show_format;
@ -441,6 +464,8 @@ extern void writeBackupCatalogConfig(FILE *out, pgBackupConfig *config);
extern void writeBackupCatalogConfigFile(pgBackupConfig *config);
extern pgBackupConfig* readBackupCatalogConfigFile(void);
extern uint32 get_config_xlog_seg_size(void);
/* in show.c */
extern int do_show(time_t requested_backup_id);
@ -540,23 +565,23 @@ extern void get_wal_file(const char *from_path, const char *to_path);
extern bool calc_file_checksum(pgFile *file);
/* parsexlog.c */
extern void extractPageMap(const char *datadir,
XLogRecPtr startpoint,
TimeLineID tli,
XLogRecPtr endpoint,
parray *backup_files_list);
extern void extractPageMap(const char *archivedir,
TimeLineID tli, uint32 seg_size,
XLogRecPtr startpoint, XLogRecPtr endpoint,
parray *files);
extern void validate_wal(pgBackup *backup,
const char *archivedir,
time_t target_time,
TransactionId target_xid,
XLogRecPtr target_lsn,
TimeLineID tli);
TimeLineID tli, uint32 seg_size);
extern bool read_recovery_info(const char *archivedir, TimeLineID tli,
uint32 seg_size,
XLogRecPtr start_lsn, XLogRecPtr stop_lsn,
time_t *recovery_time,
TransactionId *recovery_xid);
extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn,
TimeLineID target_tli);
TimeLineID target_tli, uint32 seg_size);
/* in util.c */
extern TimeLineID get_current_timeline(bool safe);
@ -564,6 +589,7 @@ extern XLogRecPtr get_checkpoint_location(PGconn *conn);
extern uint64 get_system_identifier(char *pgdata);
extern uint64 get_remote_system_identifier(PGconn *conn);
extern uint32 get_data_checksum_version(bool safe);
extern uint32 get_xlog_seg_size(char *pgdata_path);
extern void sanityChecks(void);
extern void time2iso(char *buf, size_t len, time_t time);

View File

@ -318,8 +318,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
* because it's needed to form the name of xlog file.
*/
validate_wal(dest_backup, arclog_path, rt->recovery_target_time,
rt->recovery_target_xid, rt->recovery_target_lsn,
base_full_backup->tli);
rt->recovery_target_xid, rt->recovery_target_lsn,
base_full_backup->tli, xlog_seg_size);
}
/* Orphinize every OK descendant of corrupted backup */
else

View File

@ -13,6 +13,9 @@
#include <time.h>
#include "storage/bufpage.h"
#if PG_VERSION_NUM >= 110000
#include "streamutil.h"
#endif
const char *
base36enc(long unsigned int value)
@ -212,6 +215,27 @@ get_remote_system_identifier(PGconn *conn)
#endif
}
uint32
get_xlog_seg_size(char *pgdata_path)
{
#if PG_VERSION_NUM >= 110000
ControlFileData ControlFile;
char *buffer;
size_t size;
/* First fetch file... */
buffer = slurpFile(pgdata_path, "global/pg_control", &size, false);
if (buffer == NULL)
return 0;
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
return ControlFile.xlog_seg_size;
#else
return (uint32) XLOG_SEG_SIZE;
#endif
}
uint32
get_data_checksum_version(bool safe)
{

View File

@ -91,11 +91,6 @@ static const unit_conversion memory_unit_conversion_table[] =
{"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)},
{"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)},
{"TB", OPTION_UNIT_XSEGS, (1024 * 1024 * 1024) / (XLOG_SEG_SIZE / 1024)},
{"GB", OPTION_UNIT_XSEGS, (1024 * 1024) / (XLOG_SEG_SIZE / 1024)},
{"MB", OPTION_UNIT_XSEGS, -(XLOG_SEG_SIZE / (1024 * 1024))},
{"kB", OPTION_UNIT_XSEGS, -(XLOG_SEG_SIZE / 1024)},
{""} /* end of table marker */
};
@ -1140,7 +1135,7 @@ key_equals(const char *lhs, const char *rhs)
* Return number of parsed options
*/
int
pgut_readopt(const char *path, pgut_option options[], int elevel)
pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict)
{
FILE *fp;
char buf[1024];
@ -1180,7 +1175,7 @@ pgut_readopt(const char *path, pgut_option options[], int elevel)
break;
}
}
if (!options[i].type)
if (strict && !options[i].type)
elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path);
}
}

View File

@ -111,7 +111,8 @@ extern bool in_cleanup;
extern bool in_password; /* User prompts password */
extern int pgut_getopt(int argc, char **argv, pgut_option options[]);
extern int pgut_readopt(const char *path, pgut_option options[], int elevel);
extern int pgut_readopt(const char *path, pgut_option options[], int elevel,
bool strict);
extern void pgut_getopt_env(pgut_option options[]);
extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata);
extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata);

View File

@ -259,6 +259,8 @@ do_validate_all(void)
instance_name = dent->d_name;
sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name);
sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name);
xlog_seg_size = get_config_xlog_seg_size();
do_validate_instance();
}
}
@ -381,7 +383,7 @@ do_validate_instance(void)
/* Validate corresponding WAL files */
if (current_backup->status == BACKUP_STATUS_OK)
validate_wal(current_backup, arclog_path, 0,
0, 0, base_full_backup->tli);
0, 0, base_full_backup->tli, xlog_seg_size);
/*
* Mark every descendant of corrupted backup as orphan
@ -468,7 +470,8 @@ do_validate_instance(void)
//tmp_backup = find_parent_full_backup(dest_backup);
/* Revalidation successful, validate corresponding WAL files */
validate_wal(backup, arclog_path, 0,
0, 0, current_backup->tli);
0, 0, current_backup->tli,
xlog_seg_size);
}
}