db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Merge branch 'master' into issue_324

Browse Source
This commit is contained in:
Grigory Smolkin
2021-03-26 00:37:53 +03:00
parent 8a3dfd5e67 731d19089d
commit 88c1312edd
26 changed files with 851 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pgprobackup.xml
+10 -9
View File
@@ -155,7 +155,7 @@ doc/src/sgml/pgprobackup.sgml
recovery of <productname>PostgreSQL</productname> database clusters.
It is designed to perform periodic backups of the <productname>PostgreSQL</productname>
instance that enable you to restore the server in case of a failure.
<application>pg_probackup</application> supports <productname>PostgreSQL</productname> 9.5 or higher.
<application>pg_probackup</application> supports PostgreSQL 9.5 or higher.
</para>
<itemizedlist spacing="compact">
@@ -389,7 +389,7 @@ doc/src/sgml/pgprobackup.sgml
<itemizedlist spacing="compact">
<listitem>
<para>
<application>pg_probackup</application> only supports <productname>PostgreSQL</productname> 9.5 and higher.
<application>pg_probackup</application> only supports PostgreSQL 9.5 and higher.
</para>
</listitem>
<listitem>
@@ -410,7 +410,7 @@ doc/src/sgml/pgprobackup.sgml
</listitem>
<listitem>
<para>
For PostgreSQL 9.5, functions
For <productname>PostgreSQL</productname> 9.5, functions
<function>pg_create_restore_point(text)</function> and
<function>pg_switch_xlog()</function> can be executed only if
the backup role is a superuser, so backup of a
@@ -599,7 +599,7 @@ pg_probackup add-instance -B <replaceable>backup_dir</replaceable> -D <replaceab
connection</emphasis> to the <productname>PostgreSQL</productname> server:
</para>
<para>
For PostgreSQL 9.5:
For <productname>PostgreSQL</productname> 9.5:
</para>
<programlisting>
BEGIN;
@@ -1202,8 +1202,9 @@ CREATE EXTENSION ptrack;
together, which leads to false-positive results when tracking changed
blocks and increases the incremental backup size as unchanged blocks
can also be copied into the incremental backup.
Setting <varname>ptrack.map_size</varname> to a higher value
does not affect PTRACK operation. The maximum allowed value is 1024.
Setting <varname>ptrack.map_size</varname> to a higher value does not
affect PTRACK operation, but it is not recommended to set this parameter
to a value higher than 1024.
</para>
</listitem>
</orderedlist>
@@ -1711,7 +1712,7 @@ pg_probackup restore -B <replaceable>backup_dir</replaceable> --instance <replac
<para>
The speed of restore from backup can be significantly improved
by replacing only invalid and changed pages in already
existing PostgreSQL data directory using
existing <productname>PostgreSQL</productname> data directory using
<link linkend="pbk-incremental-restore-options">incremental
restore options</link> with the <xref linkend="pbk-restore"/>
command.
@@ -1874,11 +1875,11 @@ pg_probackup restore -B <replaceable>backup_dir</replaceable> --instance <replac
</note>
<note>
<para>
Due to recovery specifics of PostgreSQL versions earlier than 12,
Due to recovery specifics of <productname>PostgreSQL</productname> versions earlier than 12,
it is advisable that you set the
<ulink url="https://postgrespro.com/docs/postgresql/current/runtime-config-replication.html#GUC-HOT-STANDBY">hot_standby</ulink>
parameter to <literal>off</literal> when running partial
restore of a PostgreSQL cluster of version earlier than 12.
restore of a <productname>PostgreSQL</productname> cluster of version earlier than 12.
Otherwise the recovery may fail.
</para>
</note>
src/backup.c
+30 -18
View File
@@ -56,7 +56,7 @@ static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNode
static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli,
bool in_prev_segment, bool segment_only,
int timeout_elevel, bool in_stream_dir);
int timeout_elevel, bool in_stream_dir, pgBackup *backup);
static void check_external_for_tablespaces(parray *external_list,
PGconn *backup_conn);
@@ -165,8 +165,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
"trying to look up on previous timelines",
current.tli);
/* TODO: use read_timeline_history */
tli_list = catalog_get_timelines(&instance_config);
tli_list = get_history_streaming(&instance_config.conn_opt, current.tli, backup_list);
if (!tli_list)
/* fallback to using archive */
tli_list = catalog_get_timelines(&instance_config);
if (parray_num(tli_list) == 0)
elog(WARNING, "Cannot find valid backup on previous timelines, "
@@ -268,7 +270,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
* Because WAL streaming will start after pg_start_backup() in stream
* mode.
*/
wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false);
wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false, &current);
}
/* start stream replication */
@@ -279,6 +281,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
start_WAL_streaming(backup_conn, dst_backup_path, &instance_config.conn_opt,
current.start_lsn, current.tli);
/* Make sure that WAL streaming is working
* PAGE backup in stream mode is waited twice, first for
* segment in WAL archive and then for streamed segment
*/
wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, true, &current);
}
/* initialize backup's file list */
@@ -1262,7 +1270,7 @@ pg_is_superuser(PGconn *conn)
static XLogRecPtr
wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli,
bool in_prev_segment, bool segment_only,
int timeout_elevel, bool in_stream_dir)
int timeout_elevel, bool in_stream_dir, pgBackup *backup)
{
XLogSegNo targetSegNo;
char pg_wal_dir[MAXPGPATH];
@@ -1294,15 +1302,14 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli,
*/
if (in_stream_dir)
{
pgBackupGetPath2(&current, pg_wal_dir, lengthof(pg_wal_dir),
DATABASE_DIR, PG_XLOG_DIR);
join_path_components(pg_wal_dir, backup->database_dir, PG_XLOG_DIR);
join_path_components(wal_segment_path, pg_wal_dir, wal_segment);
wal_segment_dir = pg_wal_dir;
}
else
{
join_path_components(wal_segment_path, arclog_path, wal_segment);
wal_segment_dir = arclog_path;
wal_segment_dir = arclog_path; /* global var */
}
/* TODO: remove this in 3.0 (it is a cludge against some old bug with archive_timeout) */
@@ -1394,7 +1401,7 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli,
sleep(1);
if (interrupted)
elog(ERROR, "Interrupted during waiting for WAL archiving");
elog(ERROR, "Interrupted during waiting for WAL %s", in_stream_dir ? "streaming" : "archiving");
try_count++;
/* Inform user if WAL segment is absent in first attempt */
@@ -1418,9 +1425,10 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli,
{
if (file_exists)
elog(timeout_elevel, "WAL segment %s was %s, "
"but target LSN %X/%X could not be archived in %d seconds",
"but target LSN %X/%X could not be %s in %d seconds",
wal_segment, wal_delivery_str,
(uint32) (target_lsn >> 32), (uint32) target_lsn, timeout);
(uint32) (target_lsn >> 32), (uint32) target_lsn,
wal_delivery_str, timeout);
/* If WAL segment doesn't exist or we wait for previous segment */
else
elog(timeout_elevel,
@@ -1572,8 +1580,13 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
*/
if (pg_stop_backup_is_sent && !in_cleanup)
{
int timeout = ARCHIVE_TIMEOUT_DEFAULT;
res = NULL;
/* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */
if (instance_config.archive_timeout > 0)
timeout = instance_config.archive_timeout;
while (1)
{
if (!PQconsumeInput(conn))
@@ -1598,11 +1611,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
* If postgres haven't answered in archive_timeout seconds,
* send an interrupt.
*/
if (pg_stop_backup_timeout > instance_config.archive_timeout)
if (pg_stop_backup_timeout > timeout)
{
pgut_cancel(conn);
elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it",
instance_config.archive_timeout);
elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout);
}
}
else
@@ -1701,7 +1713,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
{
/* Wait for segment with current stop_lsn, it is ok for it to never arrive */
wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli,
false, true, WARNING, stream_wal);
false, true, WARNING, stream_wal, backup);
/* Get the first record in segment with current stop_lsn */
lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli,
@@ -1729,7 +1741,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
* because previous record can be the contrecord.
*/
lsn_tmp = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli,
true, false, ERROR, stream_wal);
true, false, ERROR, stream_wal, backup);
/* sanity */
if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp))
@@ -1743,7 +1755,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
{
/* Wait for segment with current stop_lsn */
wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli,
false, true, ERROR, stream_wal);
false, true, ERROR, stream_wal, backup);
/* Get the next closest record in segment with current stop_lsn */
lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli,
@@ -1872,7 +1884,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
*/
if (!stop_lsn_exists)
stop_backup_lsn = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli,
false, false, ERROR, stream_wal);
false, false, ERROR, stream_wal, backup);
if (stream_wal)
{
src/catalog.c
+6 -11
View File
@@ -48,7 +48,7 @@ typedef struct LockInfo
bool exclusive;
} LockInfo;
static timelineInfo *
timelineInfo *
timelineInfoNew(TimeLineID tli)
{
timelineInfo *tlinfo = (timelineInfo *) pgut_malloc(sizeof(timelineInfo));
@@ -74,7 +74,8 @@ timelineInfoFree(void *tliInfo)
if (tli->backups)
{
parray_walk(tli->backups, pgBackupFree);
/* backups themselves should freed separately */
// parray_walk(tli->backups, pgBackupFree);
parray_free(tli->backups);
}
@@ -972,17 +973,11 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id)
continue;
}
parray_append(backups, backup);
if (errno && errno != ENOENT)
{
elog(WARNING, "cannot read data directory \"%s\": %s",
data_ent->d_name, strerror(errno));
goto err_proc;
}
}
if (errno)
{
elog(WARNING, "cannot read backup root directory \"%s\": %s",
elog(WARNING, "Cannot read backup root directory \"%s\": %s",
backup_instance_path, strerror(errno));
goto err_proc;
}
@@ -2462,7 +2457,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
{
len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers);
len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc);
len += sprintf(line+len, ",\"hdr_off\":\"%li\"", file->hdr_off);
len += sprintf(line+len, ",\"hdr_off\":\"%llu\"", file->hdr_off);
len += sprintf(line+len, ",\"hdr_size\":\"%i\"", file->hdr_size);
}
src/configure.c
+16 -3
View File
@@ -299,6 +299,7 @@ do_set_config(bool missing_ok)
for (i = 0; instance_options[i].type; i++)
{
int rc = 0;
ConfigOption *opt = &instance_options[i];
char *value;
@@ -319,13 +320,25 @@ do_set_config(bool missing_ok)
}
if (strchr(value, ' '))
fprintf(fp, "%s = '%s'\n", opt->lname, value);
rc = fprintf(fp, "%s = '%s'\n", opt->lname, value);
else
fprintf(fp, "%s = %s\n", opt->lname, value);
rc = fprintf(fp, "%s = %s\n", opt->lname, value);
if (rc < 0)
elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp);
pfree(value);
}
fclose(fp);
if (ferror(fp) || fflush(fp))
elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp);
if (fclose(fp))
elog(ERROR, "Cannot close configuration file: \"%s\"", path_temp);
if (fio_sync(path_temp, FIO_LOCAL_HOST) != 0)
elog(ERROR, "Failed to sync temp configuration file \"%s\": %s",
path_temp, strerror(errno));
if (rename(path_temp, path) < 0)
{
src/data.c
+5 -5
View File
@@ -2160,7 +2160,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b
if (fseek(in, file->hdr_off, SEEK_SET))
{
elog(strict ? ERROR : WARNING, "Cannot seek to position %lu in page header map \"%s\": %s",
elog(strict ? ERROR : WARNING, "Cannot seek to position %llu in page header map \"%s\": %s",
file->hdr_off, hdr_map->path, strerror(errno));
goto cleanup;
}
@@ -2177,7 +2177,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b
if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size)
{
elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %li len: %i \"%s\": %s",
elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %llu len: %i \"%s\": %s",
file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno));
goto cleanup;
}
@@ -2208,7 +2208,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b
if (hdr_crc != file->hdr_crc)
{
elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" "
"offset: %lu, len: %lu, current: %u, expected: %u",
"offset: %llu, len: %lu, current: %u, expected: %u",
file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc);
goto cleanup;
}
@@ -2268,7 +2268,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
{
elog(LOG, "Creating page header map \"%s\"", map_path);
hdr_map->fp = fopen(map_path, PG_BINARY_W);
hdr_map->fp = fopen(map_path, "a");
if (hdr_map->fp == NULL)
elog(ERROR, "Cannot open header file \"%s\": %s",
map_path, strerror(errno));
@@ -2297,7 +2297,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
file->rel_path, z_len);
}
elog(VERBOSE, "Writing headers for file \"%s\" offset: %li, len: %i, crc: %u",
elog(VERBOSE, "Writing headers for file \"%s\" offset: %llu, len: %i, crc: %u",
file->rel_path, file->hdr_off, z_len, file->hdr_crc);
if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len)
src/pg_probackup.h
+13 -8
View File
@@ -208,6 +208,8 @@ do { \
FIN_TRADITIONAL_CRC32(crc); \
} while (0)
#define pg_off_t unsigned long long
/* Information about single file (or dir) in backup */
typedef struct pgFile
@@ -249,8 +251,8 @@ typedef struct pgFile
/* Coordinates in header map */
int n_headers; /* number of blocks in the data file in backup */
pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */
off_t hdr_off; /* offset in header map */
int hdr_size; /* offset in header map */
pg_off_t hdr_off; /* offset in header map */
int hdr_size; /* length of headers */
} pgFile;
typedef struct page_map_entry
@@ -306,7 +308,7 @@ typedef enum ShowFormat
#define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */
#define FILE_NOT_FOUND (-2) /* file disappeared during backup */
#define BLOCKNUM_INVALID (-1)
#define PROGRAM_VERSION "2.4.10"
#define PROGRAM_VERSION "2.4.11"
/* update when remote agent API or behaviour changes */
#define AGENT_PROTOCOL_VERSION 20409
@@ -406,11 +408,11 @@ typedef struct PGNodeInfo
/* structure used for access to block header map */
typedef struct HeaderMap
{
char path[MAXPGPATH];
char path_tmp[MAXPGPATH]; /* used only in merge */
FILE *fp; /* used only for writing */
char *buf; /* buffer */
off_t offset; /* current position in fp */
char path[MAXPGPATH];
char path_tmp[MAXPGPATH]; /* used only in merge */
FILE *fp; /* used only for writing */
char *buf; /* buffer */
pg_off_t offset; /* current position in fp */
pthread_mutex_t mutex;
} HeaderMap;
@@ -895,6 +897,8 @@ extern int validate_one_page(Page page, BlockNumber absolute_blkno,
uint32 checksum_version);
extern bool validate_tablespace_map(pgBackup *backup);
extern parray* get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list);
/* return codes for validate_one_page */
/* TODO: use enum */
#define PAGE_IS_VALID (-1)
@@ -924,6 +928,7 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
extern pgBackup *get_multi_timeline_parent(parray *backup_list, parray *tli_list,
TimeLineID current_tli, time_t current_start_time,
InstanceConfig *instance);
extern timelineInfo *timelineInfoNew(TimeLineID tli);
extern void timelineInfoFree(void *tliInfo);
extern parray *catalog_get_timelines(InstanceConfig *instance);
extern void do_set_backup(const char *instance_name, time_t backup_id,
src/stream.c
+202 -1
View File
@@ -10,6 +10,7 @@
#include "pg_probackup.h"
#include "receivelog.h"
#include "streamutil.h"
#include "access/timeline.h"
#include <time.h>
#include <unistd.h>
@@ -69,6 +70,7 @@ static void add_walsegment_to_filelist(parray *filelist, uint32 timeline,
uint32 xlog_seg_size);
static void add_history_file_to_filelist(parray *filelist, uint32 timeline,
char *basedir);
static parray* parse_tli_history_buffer(char *history, TimeLineID tli);
/*
* Run IDENTIFY_SYSTEM through a given connection and
@@ -353,6 +355,204 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished)
/* --- External API --- */
/*
* Maybe add a StreamOptions struct ?
* Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it.
*/
parray*
get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list)
{
PGresult *res;
PGconn *conn;
char *history;
char query[128];
parray *result = NULL;
parray *tli_list = NULL;
timelineInfo *tlinfo = NULL;
int i,j;
snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", tli);
/*
* Connect in replication mode to the server.
*/
conn = pgut_connect_replication(conn_opt->pghost,
conn_opt->pgport,
conn_opt->pgdatabase,
conn_opt->pguser,
false);
if (!conn)
return NULL;
res = PQexec(conn, query);
PQfinish(conn);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
elog(WARNING, "Could not send replication command \"%s\": %s",
query, PQresultErrorMessage(res));
PQclear(res);
return NULL;
}
/*
* The response to TIMELINE_HISTORY is a single row result set
* with two fields: filename and content
*/
if (PQnfields(res) != 2 || PQntuples(res) != 1)
{
elog(WARNING, "Unexpected response to TIMELINE_HISTORY command: "
"got %d rows and %d fields, expected %d rows and %d fields",
PQntuples(res), PQnfields(res), 1, 2);
PQclear(res);
return NULL;
}
history = pgut_strdup(PQgetvalue(res, 0, 1));
result = parse_tli_history_buffer(history, tli);
/* some cleanup */
pg_free(history);
PQclear(res);
if (result)
tlinfo = timelineInfoNew(tli);
else
return NULL;
/* transform TimeLineHistoryEntry into timelineInfo */
for (i = parray_num(result) -1; i >= 0; i--)
{
TimeLineHistoryEntry *tln = (TimeLineHistoryEntry *) parray_get(result, i);
tlinfo->parent_tli = tln->tli;
tlinfo->switchpoint = tln->end;
if (!tli_list)
tli_list = parray_new();
parray_append(tli_list, tlinfo);
/* Next tli */
tlinfo = timelineInfoNew(tln->tli);
/* oldest tli */
if (i == 0)
{
tlinfo->tli = tln->tli;
tlinfo->parent_tli = 0;
tlinfo->switchpoint = 0;
parray_append(tli_list, tlinfo);
}
}
/* link parent to child */
for (i = 0; i < parray_num(tli_list); i++)
{
timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i);
for (j = 0; j < parray_num(tli_list); j++)
{
timelineInfo *tlinfo_parent = (timelineInfo *) parray_get(tli_list, j);
if (tlinfo->parent_tli == tlinfo_parent->tli)
{
tlinfo->parent_link = tlinfo_parent;
break;
}
}
}
/* add backups to each timeline info */
for (i = 0; i < parray_num(tli_list); i++)
{
timelineInfo *tlinfo = parray_get(tli_list, i);
for (j = 0; j < parray_num(backup_list); j++)
{
pgBackup *backup = parray_get(backup_list, j);
if (tlinfo->tli == backup->tli)
{
if (tlinfo->backups == NULL)
tlinfo->backups = parray_new();
parray_append(tlinfo->backups, backup);
}
}
}
/* cleanup */
parray_walk(result, pg_free);
pg_free(result);
return tli_list;
}
parray*
parse_tli_history_buffer(char *history, TimeLineID tli)
{
char *curLine = history;
TimeLineHistoryEntry *entry;
TimeLineHistoryEntry *last_timeline = NULL;
parray *result = NULL;
/* Parse timeline history buffer string by string */
while (curLine)
{
char tempStr[1024];
char *nextLine = strchr(curLine, '\n');
int curLineLen = nextLine ? (nextLine-curLine) : strlen(curLine);
memcpy(tempStr, curLine, curLineLen);
tempStr[curLineLen] = '\0'; // NUL-terminate!
curLine = nextLine ? (nextLine+1) : NULL;
if (curLineLen > 0)
{
char *ptr;
TimeLineID tli;
uint32 switchpoint_hi;
uint32 switchpoint_lo;
int nfields;
for (ptr = tempStr; *ptr; ptr++)
{
if (!isspace((unsigned char) *ptr))
break;
}
if (*ptr == '\0' || *ptr == '#')
continue;
nfields = sscanf(tempStr, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo);
if (nfields < 1)
{
/* expect a numeric timeline ID as first field of line */
elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a numeric timeline ID.", tempStr);
}
if (nfields != 3)
elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a transaction log switchpoint location.", tempStr);
if (last_timeline && tli <= last_timeline->tli)
elog(ERROR, "Timeline IDs must be in increasing sequence: \"%s\"", tempStr);
entry = pgut_new(TimeLineHistoryEntry);
entry->tli = tli;
entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo;
last_timeline = entry;
/* Build list with newest item first */
if (!result)
result = parray_new();
parray_append(result, entry);
/* we ignore the remainder of each line */
}
}
return result;
}
/*
* Maybe add a StreamOptions struct ?
* Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it.
@@ -374,7 +574,8 @@ start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOption
stream_thread_arg.conn = pgut_connect_replication(conn_opt->pghost,
conn_opt->pgport,
conn_opt->pgdatabase,
conn_opt->pguser);
conn_opt->pguser,
true);
/* sanity check*/
IdentifySystem(&stream_thread_arg);
src/utils/logger.c
+1
View File
@@ -169,6 +169,7 @@ exit_if_necessary(int elevel)
{
/* Interrupt other possible routines */
thread_interrupted = true;
interrupted = true;
#ifdef WIN32
ExitThread(elevel);
#else
src/utils/pgut.c
+3 -2
View File
@@ -270,7 +270,8 @@ pgut_connect(const char *host, const char *port,
PGconn *
pgut_connect_replication(const char *host, const char *port,
const char *dbname, const char *username)
const char *dbname, const char *username,
bool strict)
{
PGconn *tmpconn;
int argcount = 7; /* dbname, replication, fallback_app_name,
@@ -356,7 +357,7 @@ pgut_connect_replication(const char *host, const char *port,
continue;
}
elog(ERROR, "could not connect to database %s: %s",
elog(strict ? ERROR : WARNING, "could not connect to database %s: %s",
dbname, PQerrorMessage(tmpconn));
PQfinish(tmpconn);
free(values);
src/utils/pgut.h
+2 -2
View File
@@ -40,8 +40,8 @@ extern char *pgut_get_conninfo_string(PGconn *conn);
extern PGconn *pgut_connect(const char *host, const char *port,
const char *dbname, const char *username);
extern PGconn *pgut_connect_replication(const char *host, const char *port,
const char *dbname,
const char *username);
const char *dbname, const char *username,
bool strict);
extern void pgut_disconnect(PGconn *conn);
extern void pgut_disconnect_callback(bool fatal, void *userdata);
extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams,
src/validate.c
+3 -1
View File
@@ -721,6 +721,8 @@ validate_tablespace_map(pgBackup *backup)
pgFile **tablespace_map = NULL;
pg_crc32 crc;
parray *files = get_backup_filelist(backup, true);
bool use_crc32c = parse_program_version(backup->program_version) <= 20021 ||
parse_program_version(backup->program_version) >= 20025;
parray_qsort(files, pgFileCompareRelPathWithExternal);
join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE);
@@ -743,7 +745,7 @@ validate_tablespace_map(pgBackup *backup)
map_path, base36enc(backup->backup_id));
/* check tablespace map checksumms */
crc = pgFileGetCRC(map_path, true, false);
crc = pgFileGetCRC(map_path, use_crc32c, false);
if ((*tablespace_map)->crc != crc)
elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, "
tests/archive.py
+86 -3
View File
@@ -981,7 +981,91 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
self.backup_node(backup_dir, 'master', replica)
# Clean after yourself
self.del_test_dir(module_name, fname, nodes=[master, replica])
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_concurrent_archiving(self):
"""
Concurrent archiving from master, replica and cascade replica
https://github.com/postgrespro/pg_probackup/issues/327
For PG >= 11 it is expected to pass this test
"""
if self.pg_config_version < self.version_to_num('11.0'):
return unittest.skip('You need PostgreSQL >= 11 for this test')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
master = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'master'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', master)
self.set_archiving(backup_dir, 'node', master, replica=True)
master.slow_start()
master.pgbench_init(scale=10)
# TAKE FULL ARCHIVE BACKUP FROM MASTER
self.backup_node(backup_dir, 'node', master)
# Settings for Replica
replica = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'replica'))
replica.cleanup()
self.restore_node(backup_dir, 'node', replica)
self.set_replica(master, replica, synchronous=True)
self.set_archiving(backup_dir, 'node', replica, replica=True)
self.set_auto_conf(replica, {'port': replica.port})
replica.slow_start(replica=True)
# create cascade replicas
replica1 = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'replica1'))
replica1.cleanup()
# Settings for casaced replica
self.restore_node(backup_dir, 'node', replica1)
self.set_replica(replica, replica1, synchronous=False)
self.set_auto_conf(replica1, {'port': replica1.port})
replica1.slow_start(replica=True)
# Take full backup from master
self.backup_node(backup_dir, 'node', master)
pgbench = master.pgbench(
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
options=['-T', '30', '-c', '1'])
# Take several incremental backups from master
self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate'])
self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate'])
pgbench.wait()
pgbench.stdout.close()
with open(os.path.join(master.logs_dir, 'postgresql.log'), 'r') as f:
log_content = f.read()
self.assertNotIn('different checksum', log_content)
with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f:
log_content = f.read()
self.assertNotIn('different checksum', log_content)
with open(os.path.join(replica1.logs_dir, 'postgresql.log'), 'r') as f:
log_content = f.read()
self.assertNotIn('different checksum', log_content)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
@@ -2016,8 +2100,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
pg_receivexlog.kill()
self.del_test_dir(
module_name, fname, [node, replica, node_restored])
self.del_test_dir(module_name, fname)
@unittest.skip("skip")
def test_multi_timeline_recovery_prefetching(self):
tests/backup.py
+340 -6
View File
@@ -1065,7 +1065,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname, nodes=[node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_tablespace_handling_1(self):
@@ -1599,7 +1599,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
options=['--stream', '--slot=slot_1', '--temp-slot'])
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_backup_concurrent_drop_table(self):
@@ -1645,7 +1645,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
self.assertEqual(show_backup['status'], "OK")
# Clean after yourself
self.del_test_dir(module_name, fname, nodes=[node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_pg_11_adjusted_wal_segment_size(self):
@@ -1930,7 +1930,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
os.chmod(full_path, 700)
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_basic_missing_dir_permissions(self):
@@ -1973,7 +1973,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
os.rmdir(full_path)
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_backup_with_least_privileges_role(self):
@@ -2365,7 +2365,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
@unittest.skip("skip")
# @unittest.skip("skip")
def test_backup_with_less_privileges_role(self):
"""
check permissions correctness from documentation:
@@ -3079,3 +3079,337 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_missing_wal_segment(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'],
pg_options={'archive_timeout': '30s'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=10)
node.safe_psql(
'postgres',
'CREATE DATABASE backupdb')
# get segments in pg_wal, sort then and remove all but the latest
pg_wal_dir = os.path.join(node.data_dir, 'pg_wal')
if node.major_version >= 10:
pg_wal_dir = os.path.join(node.data_dir, 'pg_wal')
else:
pg_wal_dir = os.path.join(node.data_dir, 'pg_xlog')
# Full backup in streaming mode
gdb = self.backup_node(
backup_dir, 'node', node, datname='backupdb',
options=['--stream', '--log-level-file=INFO'], gdb=True)
# break at streaming start
gdb.set_breakpoint('start_WAL_streaming')
gdb.run_until_break()
# generate some more data
node.pgbench_init(scale=3)
# remove redundant WAL segments in pg_wal
files = os.listdir(pg_wal_dir)
files.sort(reverse=True)
# leave first two files in list
del files[:2]
for filename in files:
os.remove(os.path.join(pg_wal_dir, filename))
gdb.continue_execution_until_exit()
self.assertIn(
'unexpected termination of replication stream: ERROR: requested WAL segment',
gdb.output)
self.assertIn(
'has already been removed',
gdb.output)
self.assertIn(
'ERROR: Interrupted during waiting for WAL streaming',
gdb.output)
self.assertIn(
'WARNING: backup in progress, stop backup',
gdb.output)
# TODO: check the same for PAGE backup
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_missing_replication_permission(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
# self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# FULL backup
self.backup_node(backup_dir, 'node', node, options=['--stream'])
# Create replica
replica = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'replica'))
replica.cleanup()
self.restore_node(backup_dir, 'node', replica)
# Settings for Replica
self.set_replica(node, replica)
replica.slow_start(replica=True)
node.safe_psql(
'postgres',
'CREATE DATABASE backupdb')
# PG 9.5
if self.get_version(node) < 90600:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;")
# PG 9.6
elif self.get_version(node) > 90600 and self.get_version(node) < 100000:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;")
# >= 10
else:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
if ProbackupTest.enterprise:
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup")
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup")
sleep(2)
replica.promote()
# Delta backup
try:
self.backup_node(
backup_dir, 'node', replica, backup_type='delta',
data_dir=replica.data_dir, datname='backupdb', options=['--stream', '-U', 'backup'])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because incremental backup should not be possible "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
"FATAL: must be superuser or replication role to start walsender",
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_missing_replication_permission_1(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# FULL backup
self.backup_node(backup_dir, 'node', node, options=['--stream'])
# Create replica
replica = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'replica'))
replica.cleanup()
self.restore_node(backup_dir, 'node', replica)
# Settings for Replica
self.set_replica(node, replica)
replica.slow_start(replica=True)
node.safe_psql(
'postgres',
'CREATE DATABASE backupdb')
# PG 9.5
if self.get_version(node) < 90600:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;")
# PG 9.6
elif self.get_version(node) > 90600 and self.get_version(node) < 100000:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;")
# >= 10
else:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
if ProbackupTest.enterprise:
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup")
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup")
replica.promote()
# PAGE
output = self.backup_node(
backup_dir, 'node', replica, backup_type='page',
data_dir=replica.data_dir, datname='backupdb', options=['-U', 'backup'],
return_id=False)
self.assertIn(
'WARNING: Valid backup on current timeline 2 is not found, trying to look up on previous timelines',
output)
self.assertIn(
'WARNING: could not connect to database backupdb: FATAL: must be superuser or replication role to start walsender',
output)
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/checkdb.py
+1 -1
View File
@@ -349,7 +349,7 @@ class CheckdbTest(ProbackupTest, unittest.TestCase):
log_file_content)
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_checkdb_block_validation_sanity(self):
tests/compatibility.py
+82
View File
@@ -1419,3 +1419,85 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_compatibility_tablespace(self):
"""
https://github.com/postgrespro/pg_probackup/issues/348
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node, old_binary=True)
node.slow_start()
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"], old_binary=True)
tblspace_old_path = self.get_tblspace_path(node, 'tblspace_old')
self.create_tblspace_in_node(
node, 'tblspace',
tblspc_path=tblspace_old_path)
node.safe_psql(
"postgres",
"create table t_heap_lame tablespace tblspace "
"as select 1 as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1000) i")
tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new')
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
try:
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"-T", "{0}={1}".format(
tblspace_old_path, tblspace_new_path)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because tablespace mapping is incorrect"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
'ERROR: Backup {0} has no tablespaceses, '
'nothing to remap'.format(backup_id),
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=["-j", "4", "--stream"], old_binary=True)
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"-T", "{0}={1}".format(
tblspace_old_path, tblspace_new_path)])
if self.paranoia:
pgdata = self.pgdata_content(node.data_dir)
if self.paranoia:
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/compression.py
+1 -1
View File
@@ -117,7 +117,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(delta_result, delta_result_new)
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
def test_compression_archive_zlib(self):
"""
tests/delta.py
+1 -1
View File
@@ -80,7 +80,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase):
node_restored.slow_start()
# Clean after yourself
self.del_test_dir(module_name, fname, [node, node_restored])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_delta_vacuum_truncate_1(self):
tests/expected/option_version.out
+1 -1
View File
@@ -1 +1 @@
pg_probackup 2.4.10
pg_probackup 2.4.11
tests/helpers/ptrack_helpers.py
+19 -21
View File
@@ -1,5 +1,6 @@
# you need os for unittest to work
import os
import gc
from sys import exit, argv, version_info
import subprocess
import shutil
@@ -402,7 +403,6 @@ class ProbackupTest(object):
if node.major_version >= 13:
self.set_auto_conf(
node, {}, 'postgresql.conf', ['wal_keep_segments'])
return node
def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False):
@@ -1514,15 +1514,15 @@ class ProbackupTest(object):
def get_bin_path(self, binary):
return testgres.get_bin_path(binary)
def del_test_dir(self, module_name, fname, nodes=[]):
""" Del testdir and optimistically try to del module dir"""
try:
testgres.clean_all()
except:
pass
def clean_all(self):
for o in gc.get_referrers(testgres.PostgresNode):
if o.__class__ is testgres.PostgresNode:
o.cleanup()
for node in nodes:
node.stop()
def del_test_dir(self, module_name, fname):
""" Del testdir and optimistically try to del module dir"""
self.clean_all()
shutil.rmtree(
os.path.join(
@@ -1532,10 +1532,6 @@ class ProbackupTest(object):
),
ignore_errors=True
)
try:
os.rmdir(os.path.join(self.tmp_path, module_name))
except:
pass
def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None):
""" return dict with directory content. "
@@ -1761,6 +1757,7 @@ class GdbException(Exception):
class GDBobj(ProbackupTest):
def __init__(self, cmd, verbose, attach=False):
self.verbose = verbose
self.output = ''
# Check gdb presense
try:
@@ -1802,10 +1799,8 @@ class GDBobj(ProbackupTest):
)
self.gdb_pid = self.proc.pid
# discard data from pipe,
# is there a way to do it a less derpy way?
while True:
line = self.proc.stdout.readline()
line = self.get_line()
if 'No such process' in line:
raise GdbException(line)
@@ -1815,6 +1810,11 @@ class GDBobj(ProbackupTest):
else:
break
def get_line(self):
line = self.proc.stdout.readline()
self.output += line
return line
def kill(self):
self.proc.kill()
self.proc.wait()
@@ -1936,10 +1936,8 @@ class GDBobj(ProbackupTest):
'Failed to continue execution until break.\n')
def stopped_in_breakpoint(self):
output = []
while True:
line = self.proc.stdout.readline()
output += [line]
line = self.get_line()
if self.verbose:
print(line)
if line.startswith('*stopped,reason="breakpoint-hit"'):
@@ -1956,7 +1954,7 @@ class GDBobj(ProbackupTest):
# look for command we just send
while True:
line = self.proc.stdout.readline()
line = self.get_line()
if self.verbose:
print(repr(line))
@@ -1966,7 +1964,7 @@ class GDBobj(ProbackupTest):
break
while True:
line = self.proc.stdout.readline()
line = self.get_line()
output += [line]
if self.verbose:
print(repr(line))
tests/incr_restore.py
+11 -11
View File
@@ -720,7 +720,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_incr_checksum_restore(self):
@@ -809,7 +809,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname, [node, node_1])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
@@ -898,7 +898,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname, [node, node_1])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_incr_lsn_sanity(self):
@@ -967,7 +967,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname, [node_1])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_incr_checksum_sanity(self):
@@ -1026,7 +1026,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname, [node_1])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
@@ -1561,7 +1561,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
pgbench.wait()
# Clean after yourself
self.del_test_dir(module_name, fname, [new_master, old_master])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_make_replica_via_incr_lsn_restore(self):
@@ -1634,7 +1634,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
pgbench.wait()
# Clean after yourself
self.del_test_dir(module_name, fname, [new_master, old_master])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
@@ -1878,7 +1878,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
'1')
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
@@ -2137,7 +2137,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
self.assertNotIn('PANIC', output)
# Clean after yourself
self.del_test_dir(module_name, fname, [node, node2])
self.del_test_dir(module_name, fname)
def test_incremental_partial_restore_exclude_lsn(self):
""""""
@@ -2247,7 +2247,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
self.assertNotIn('PANIC', output)
# Clean after yourself
self.del_test_dir(module_name, fname, [node2])
self.del_test_dir(module_name, fname)
def test_incremental_partial_restore_exclude_tablespace_checksum(self):
""""""
@@ -2391,7 +2391,7 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase):
self.assertNotIn('PANIC', output)
# Clean after yourself
self.del_test_dir(module_name, fname, [node2])
self.del_test_dir(module_name, fname)
def test_incremental_pg_filenode_map(self):
"""
tests/merge.py
+7 -7
View File
@@ -101,7 +101,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
self.assertEqual(count1, count2)
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
def test_merge_compressed_backups(self):
"""
@@ -2245,7 +2245,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_smart_merge(self):
@@ -2305,7 +2305,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
logfile_content = f.read()
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
def test_idempotent_merge(self):
"""
@@ -2380,7 +2380,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
self.assertEqual(
page_id_2, self.show_pb(backup_dir, 'node')[0]['id'])
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
def test_merge_correct_inheritance(self):
"""
@@ -2435,7 +2435,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
page_meta['expire-time'],
self.show_pb(backup_dir, 'node', page_id)['expire-time'])
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
def test_merge_correct_inheritance_1(self):
"""
@@ -2485,7 +2485,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
'expire-time',
self.show_pb(backup_dir, 'node', page_id))
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
@@ -2604,7 +2604,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
'-d', 'postgres', '-p', str(node_restored.port)])
# Clean after yourself
self.del_test_dir(module_name, fname, [node, node_restored])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
tests/page.py
+1 -1
View File
@@ -100,7 +100,7 @@ class PageTest(ProbackupTest, unittest.TestCase):
self.assertEqual(result1, result2)
# Clean after yourself
self.del_test_dir(module_name, fname, [node, node_restored])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_page_vacuum_truncate_1(self):
tests/ptrack.py
+1 -1
View File
@@ -3148,7 +3148,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
'select 1')
# Clean after yourself
self.del_test_dir(module_name, fname, [master, replica, node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
tests/replica.py
+1 -1
View File
@@ -418,7 +418,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
options=['--archive-timeout=30s', '--stream'])
# Clean after yourself
self.del_test_dir(module_name, fname, [master, replica])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_take_backup_from_delayed_replica(self):
tests/retention.py
+4 -4
View File
@@ -1008,7 +1008,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
'FULL')
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_basic_window_merge_multiple_descendants_1(self):
@@ -1275,7 +1275,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
'--delete-expired', '--log-level-console=log'])
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_window_chains(self):
@@ -2535,7 +2535,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.validate_pb(backup_dir, 'node')
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
def test_concurrent_running_full_backup(self):
"""
@@ -2613,4 +2613,4 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
len(self.show_pb(backup_dir, 'node')),
6)
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
tests/validate.py
+4 -4
View File
@@ -298,7 +298,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
'Backup STATUS should be "ORPHAN"')
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_validate_corrupted_intermediate_backups(self):
@@ -3843,7 +3843,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
self.assertIn("WARNING: Some backups are not valid", e.message)
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
@@ -3906,7 +3906,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
self.assertIn("WARNING: Some backups are not valid", e.message)
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
@@ -3966,7 +3966,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
self.assertIn("WARNING: Some backups are not valid", e.message)
# Clean after yourself
self.del_test_dir(module_name, fname, [node])
self.del_test_dir(module_name, fname)
# validate empty backup list
# page from future during validate