db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Merge branch 'master' into issue_83

Browse Source
This commit is contained in:
Grigory Smolkin
2019-07-19 14:37:47 +03:00
parent 49163b3810 028029e899
commit 05d8ecc253
40 changed files with 2932 additions and 1250 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation.md
+922 -557
View File
File diff suppressed because it is too large Load Diff
README.md
+6 -6
View File
@@ -45,6 +45,10 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
* Remote mode is in beta stage.
* Incremental chain can span only within one timeline. So if you have backup incremental chain taken from replica and it gets promoted, you would be forced to take another FULL backup.
## Current release
[2.1.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.4)
## Installation and Setup
### Windows Installation
[Installers download link](https://oc.postgrespro.ru/index.php/s/CGsjXlc5NmhRI0L)
@@ -81,7 +85,7 @@ yum install pg_probackup-{11,10,9.6,9.5}-debuginfo
yumdownloader --source pg_probackup-{11,10,9.6,9.5}
```
Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup).
Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup).
## Building from source
### Linux
@@ -104,13 +108,9 @@ SET PATH=%PATH%;C:\msys64\usr\bin
gen_probackup_project.pl C:\path_to_postgresql_source_tree
```
## Current release
[2.1.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.3)
## Documentation
Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup).
Currently the latest documentation can be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md) and [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup).
## Licence
src/backup.c
+197 -188
View File
@@ -78,10 +78,6 @@ static int is_ptrack_enable = false;
bool is_ptrack_support = false;
bool exclusive_backup = false;
/* PostgreSQL server version from "backup_conn" */
static int server_version = 0;
static char server_version_str[100] = "";
/* Is pg_start_backup() was executed */
static bool backup_in_progress = false;
/* Is pg_stop_backup() was sent */
@@ -94,20 +90,20 @@ static void backup_cleanup(bool fatal, void *userdata);
static void *backup_files(void *arg);
static void do_backup_instance(PGconn *backup_conn);
static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo);
static void pg_start_backup(const char *label, bool smooth, pgBackup *backup,
PGconn *backup_conn, PGconn *master_conn);
PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn);
static void pg_switch_wal(PGconn *conn);
static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn);
static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo);
static int checkpoint_timeout(PGconn *backup_conn);
//static void backup_list_file(parray *files, const char *root, )
static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn,
bool wait_prev_segment);
static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup, PGconn *backup_conn);
static void make_pagemap_from_ptrack(parray* files, PGconn* backup_conn);
static void *StreamLog(void *arg);
static void IdentifySystem(StreamThreadArg *stream_thread_arg);
static void check_external_for_tablespaces(parray *external_list,
PGconn *backup_conn);
@@ -128,7 +124,8 @@ static XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn);
/* Check functions */
static bool pg_checksum_enable(PGconn *conn);
static bool pg_is_in_recovery(PGconn *conn);
static void check_server_version(PGconn *conn);
static bool pg_is_superuser(PGconn *conn);
static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo);
static void confirm_block_size(PGconn *conn, const char *name, int blcksz);
static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i);
@@ -142,7 +139,7 @@ backup_stopbackup_callback(bool fatal, void *userdata)
if (backup_in_progress)
{
elog(WARNING, "backup in progress, stop backup");
pg_stop_backup(NULL, pg_startbackup_conn); /* don't care stop_lsn on error case */
pg_stop_backup(NULL, pg_startbackup_conn, NULL); /* don't care about stop_lsn in case of error */
}
}
@@ -151,7 +148,7 @@ backup_stopbackup_callback(bool fatal, void *userdata)
* Move files from 'pgdata' to a subdirectory in 'backup_path'.
*/
static void
do_backup_instance(PGconn *backup_conn)
do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
{
int i;
char database_path[MAXPGPATH];
@@ -198,10 +195,11 @@ do_backup_instance(PGconn *backup_conn)
/* get list of backups already taken */
backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
prev_backup = catalog_get_last_data_backup(backup_list, current.tli);
prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time);
if (prev_backup == NULL)
elog(ERROR, "Valid backup on current timeline is not found. "
"Create new FULL backup before an incremental one.");
elog(ERROR, "Valid backup on current timeline %X is not found. "
"Create new FULL backup before an incremental one.",
current.tli);
pgBackupGetPath(prev_backup, prev_backup_filelist_path,
lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST);
@@ -254,8 +252,7 @@ do_backup_instance(PGconn *backup_conn)
else
pg_startbackup_conn = backup_conn;
pg_start_backup(label, smooth_checkpoint, &current,
backup_conn, pg_startbackup_conn);
pg_start_backup(label, smooth_checkpoint, &current, nodeInfo, backup_conn, pg_startbackup_conn);
/* For incremental backup check that start_lsn is not from the past */
if (current.backup_mode != BACKUP_MODE_FULL &&
@@ -293,30 +290,10 @@ do_backup_instance(PGconn *backup_conn)
instance_config.conn_opt.pgport,
instance_config.conn_opt.pgdatabase,
instance_config.conn_opt.pguser);
/* sanity */
IdentifySystem(&stream_thread_arg);
if (!CheckServerVersionForStreaming(stream_thread_arg.conn))
{
PQfinish(stream_thread_arg.conn);
/*
* Error message already written in CheckServerVersionForStreaming().
* There's no hope of recovering from a version mismatch, so don't
* retry.
*/
elog(ERROR, "Cannot continue backup because stream connect has failed.");
}
/*
* Identify server, obtaining start LSN position and current timeline ID
* at the same time, necessary if not valid data can be found in the
* existing output directory.
*/
if (!RunIdentifySystem(stream_thread_arg.conn, NULL, NULL, NULL, NULL))
{
PQfinish(stream_thread_arg.conn);
elog(ERROR, "Cannot continue backup because stream connect has failed.");
}
/* By default there are some error */
/* By default there are some error */
stream_thread_arg.ret = 1;
/* we must use startpos as start_lsn from start_backup */
stream_thread_arg.startpos = current.start_lsn;
@@ -344,6 +321,9 @@ do_backup_instance(PGconn *backup_conn)
dir_list_file(backup_files_list, parray_get(external_dirs, i),
false, true, false, i+1, FIO_DB_HOST);
/* close ssh session in main thread */
fio_disconnect();
/* Sanity check for backup_files_list, thank you, Windows:
* https://github.com/postgrespro/pg_probackup/issues/48
*/
@@ -512,6 +492,9 @@ do_backup_instance(PGconn *backup_conn)
parray_free(prev_backup_filelist);
}
/* Notify end of backup */
pg_stop_backup(&current, pg_startbackup_conn, nodeInfo);
/* In case of backup from replica >= 9.6 we must fix minRecPoint,
* First we must find pg_control in backup_files_list.
*/
@@ -520,7 +503,7 @@ do_backup_instance(PGconn *backup_conn)
char pg_control_path[MAXPGPATH];
snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s",
instance_config.pgdata, "global/pg_control");
instance_config.pgdata, XLOG_CONTROL_FILE);
for (i = 0; i < parray_num(backup_files_list); i++)
{
@@ -532,13 +515,16 @@ do_backup_instance(PGconn *backup_conn)
break;
}
}
if (!pg_control)
elog(ERROR, "Failed to find file \"%s\" in backup filelist.",
pg_control_path);
set_min_recovery_point(pg_control, database_path, current.stop_lsn);
}
/* Notify end of backup */
pg_stop_backup(&current, pg_startbackup_conn);
if (current.from_replica && !exclusive_backup)
set_min_recovery_point(pg_control, database_path, current.stop_lsn);
/* close ssh session in main thread */
fio_disconnect();
/* Add archived xlog files into the list of files of this backup */
if (stream_wal)
@@ -626,11 +612,12 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
confirm_block_size(cur_conn, "wal_block_size", XLOG_BLCKSZ);
nodeInfo->block_size = BLCKSZ;
nodeInfo->wal_block_size = XLOG_BLCKSZ;
nodeInfo->is_superuser = pg_is_superuser(cur_conn);
current.from_replica = pg_is_in_recovery(cur_conn);
/* Confirm that this server version is supported */
check_server_version(cur_conn);
check_server_version(cur_conn, nodeInfo);
if (pg_checksum_enable(cur_conn))
current.checksum_version = 1;
@@ -647,11 +634,12 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
"pg_probackup have no way to detect data block corruption without them. "
"Reinitialize PGDATA with option '--data-checksums'.");
StrNCpy(current.server_version, server_version_str,
sizeof(current.server_version));
if (nodeInfo->is_superuser)
elog(WARNING, "Current PostgreSQL role is superuser. "
"It is not recommended to run backup or checkdb as superuser.");
StrNCpy(nodeInfo->server_version, server_version_str,
sizeof(nodeInfo->server_version));
StrNCpy(current.server_version, nodeInfo->server_version_str,
sizeof(current.server_version));
return cur_conn;
}
@@ -663,16 +651,24 @@ int
do_backup(time_t start_time, bool no_validate)
{
PGconn *backup_conn = NULL;
PGNodeInfo nodeInfo;
char pretty_data_bytes[20];
/* Initialize PGInfonode */
pgNodeInit(&nodeInfo);
if (!instance_config.pgdata)
elog(ERROR, "required parameter not specified: PGDATA "
"(-D, --pgdata)");
current.compress_alg = instance_config.compress_alg;
current.compress_level = instance_config.compress_level;
/*
* setup backup_conn, do some compatibility checks and
* fill basic info about instance
*/
backup_conn = pgdata_basic_setup(instance_config.conn_opt,
&(current.nodeInfo));
backup_conn = pgdata_basic_setup(instance_config.conn_opt, &nodeInfo);
/*
* Ensure that backup directory was initialized for the same PostgreSQL
* instance we opened connection to. And that target backup database PGDATA
@@ -680,17 +676,19 @@ do_backup(time_t start_time, bool no_validate)
*/
check_system_identifiers(backup_conn, instance_config.pgdata);
elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, "
"wal-method: %s, remote: %s, replica: %s, compress-algorithm: %s, compress-level: %i",
PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(&current),
current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false",
current.from_replica ? "true" : "false", deparse_compress_alg(current.compress_alg),
current.compress_level);
/* below perform checks specific for backup command */
#if PG_VERSION_NUM >= 110000
if (!RetrieveWalSegSize(backup_conn))
elog(ERROR, "Failed to retreive wal_segment_size");
#endif
current.compress_alg = instance_config.compress_alg;
current.compress_level = instance_config.compress_level;
current.stream = stream_wal;
is_ptrack_support = pg_ptrack_support(backup_conn);
if (is_ptrack_support)
{
@@ -740,7 +738,7 @@ do_backup(time_t start_time, bool no_validate)
pgut_atexit_push(backup_cleanup, NULL);
/* backup data */
do_backup_instance(backup_conn);
do_backup_instance(backup_conn, &nodeInfo);
pgut_atexit_pop(backup_cleanup, NULL);
/* compute size of wal files of this backup stored in the archive */
@@ -756,12 +754,13 @@ do_backup(time_t start_time, bool no_validate)
current.status = BACKUP_STATUS_DONE;
write_backup(&current);
//elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "",
// current.data_bytes);
if (!no_validate)
pgBackupValidate(&current);
/* Notify user about backup size */
pretty_size(current.data_bytes, pretty_data_bytes, lengthof(pretty_data_bytes));
elog(INFO, "Backup %s real size: %s", base36enc(current.start_time), pretty_data_bytes);
if (current.status == BACKUP_STATUS_OK ||
current.status == BACKUP_STATUS_DONE)
elog(INFO, "Backup %s completed", base36enc(current.start_time));
@@ -782,34 +781,35 @@ do_backup(time_t start_time, bool no_validate)
* Confirm that this server version is supported
*/
static void
check_server_version(PGconn *conn)
check_server_version(PGconn *conn, PGNodeInfo *nodeInfo)
{
PGresult *res;
/* confirm server version */
server_version = PQserverVersion(conn);
nodeInfo->server_version = PQserverVersion(conn);
if (server_version == 0)
elog(ERROR, "Unknown server version %d", server_version);
if (nodeInfo->server_version == 0)
elog(ERROR, "Unknown server version %d", nodeInfo->server_version);
if (server_version < 100000)
sprintf(server_version_str, "%d.%d",
server_version / 10000,
(server_version / 100) % 100);
if (nodeInfo->server_version < 100000)
sprintf(nodeInfo->server_version_str, "%d.%d",
nodeInfo->server_version / 10000,
(nodeInfo->server_version / 100) % 100);
else
sprintf(server_version_str, "%d",
server_version / 10000);
sprintf(nodeInfo->server_version_str, "%d",
nodeInfo->server_version / 10000);
if (server_version < 90500)
if (nodeInfo->server_version < 90500)
elog(ERROR,
"server version is %s, must be %s or higher",
server_version_str, "9.5");
nodeInfo->server_version_str, "9.5");
if (current.from_replica && server_version < 90600)
if (current.from_replica && nodeInfo->server_version < 90600)
elog(ERROR,
"server version is %s, must be %s or higher for backup from replica",
server_version_str, "9.6");
nodeInfo->server_version_str, "9.6");
/* TODO: search pg_proc for pgpro_edition before calling */
res = pgut_execute_extended(conn, "SELECT pgpro_edition()",
0, NULL, true, true);
@@ -822,29 +822,29 @@ check_server_version(PGconn *conn)
/* It seems we connected to PostgreSQL (not Postgres Pro) */
elog(ERROR, "%s was built with Postgres Pro %s %s, "
"but connection is made with PostgreSQL %s",
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str);
else if (strcmp(server_version_str, PG_MAJORVERSION) != 0 &&
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
else if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 &&
strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0)
elog(ERROR, "%s was built with Postgres Pro %s %s, "
"but connection is made with Postgres Pro %s %s",
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION,
server_version_str, PQgetvalue(res, 0, 0));
nodeInfo->server_version_str, PQgetvalue(res, 0, 0));
#else
if (PQresultStatus(res) != PGRES_FATAL_ERROR)
/* It seems we connected to Postgres Pro (not PostgreSQL) */
elog(ERROR, "%s was built with PostgreSQL %s, "
"but connection is made with Postgres Pro %s %s",
PROGRAM_NAME, PG_MAJORVERSION,
server_version_str, PQgetvalue(res, 0, 0));
else if (strcmp(server_version_str, PG_MAJORVERSION) != 0)
nodeInfo->server_version_str, PQgetvalue(res, 0, 0));
else if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0)
elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s",
PROGRAM_NAME, PG_MAJORVERSION, server_version_str);
PROGRAM_NAME, PG_MAJORVERSION, nodeInfo->server_version_str);
#endif
PQclear(res);
/* Do exclusive backup only for PostgreSQL 9.5 */
exclusive_backup = server_version < 90600 ||
exclusive_backup = nodeInfo->server_version < 90600 ||
current.backup_mode == BACKUP_MODE_DIFF_PTRACK;
}
@@ -879,6 +879,7 @@ check_system_identifiers(PGconn *conn, char *pgdata)
elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", "
"but connected instance system id is " UINT64_FORMAT,
instance_config.system_identifier, system_id_conn);
if (system_id_pgdata != instance_config.system_identifier)
elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", "
"but target backup directory system id is " UINT64_FORMAT,
@@ -914,7 +915,7 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz)
*/
static void
pg_start_backup(const char *label, bool smooth, pgBackup *backup,
PGconn *backup_conn, PGconn *pg_startbackup_conn)
PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *pg_startbackup_conn)
{
PGresult *res;
const char *params[2];
@@ -955,11 +956,14 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
PQclear(res);
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE &&
(!(backup->from_replica && !exclusive_backup)))
!backup->from_replica &&
!(nodeInfo->server_version < 90600 &&
!nodeInfo->is_superuser))
/*
* Switch to a new WAL segment. It is necessary to get archived WAL
* segment, which includes start LSN of current backup.
* Don`t do this for replica backups unless it`s PG 9.5
* Don`t do this for replica backups and for PG 9.5 if pguser is not superuser
* (because in 9.5 only superuser can switch WAL)
*/
pg_switch_wal(conn);
@@ -974,16 +978,11 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
else if (!stream_wal)
/* ...for others wait for previous segment */
wait_wal_lsn(backup->start_lsn, true, true);
/* In case of backup from replica for PostgreSQL 9.5
* wait for start_lsn to be replayed by replica
*/
if (backup->from_replica && exclusive_backup)
wait_replica_wal_lsn(backup->start_lsn, true, backup_conn);
}
/*
* Switch to a new WAL segment. It should be called only for master.
* For PG 9.5 it should be called only if pguser is superuser.
*/
static void
pg_switch_wal(PGconn *conn)
@@ -995,9 +994,9 @@ pg_switch_wal(PGconn *conn)
PQclear(res);
#if PG_VERSION_NUM >= 100000
res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_wal()", 0, NULL);
res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_wal()", 0, NULL);
#else
res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_xlog()", 0, NULL);
res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_xlog()", 0, NULL);
#endif
PQclear(res);
@@ -1070,13 +1069,13 @@ pg_checksum_enable(PGconn *conn)
res_db = pgut_execute(conn, "SHOW data_checksums", 0, NULL);
if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0)
if (strcmp(PQgetvalue(res_db, 0, 0), "on") == 0)
{
PQclear(res_db);
return false;
return true;
}
PQclear(res_db);
return true;
return false;
}
/* Check if target instance is replica */
@@ -1096,6 +1095,24 @@ pg_is_in_recovery(PGconn *conn)
return false;
}
/* Check if current PostgreSQL role is superuser */
static bool
pg_is_superuser(PGconn *conn)
{
PGresult *res;
res = pgut_execute(conn, "SELECT pg_catalog.current_setting('is_superuser')", 0, NULL);
if (strcmp(PQgetvalue(res, 0, 0), "on") == 0)
{
PQclear(res);
return true;
}
PQclear(res);
return false;
}
/* Clear ptrack files in all databases of the instance we connected to */
static void
pg_ptrack_clear(PGconn *backup_conn)
@@ -1469,80 +1486,12 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment)
}
}
/*
* Wait for target 'lsn' on replica instance from master.
*/
static void
wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup,
PGconn *backup_conn)
{
uint32 try_count = 0;
while (true)
{
XLogRecPtr replica_lsn;
/*
* For lsn from pg_start_backup() we need it to be replayed on replica's
* data.
*/
if (is_start_backup)
{
replica_lsn = get_checkpoint_location(backup_conn);
}
/*
* For lsn from pg_stop_backup() we need it only to be received by
* replica and fsync()'ed on WAL segment.
*/
else
{
PGresult *res;
uint32 lsn_hi;
uint32 lsn_lo;
#if PG_VERSION_NUM >= 100000
res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_receive_lsn()",
0, NULL);
#else
res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_receive_location()",
0, NULL);
#endif
/* Extract LSN from result */
XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo);
/* Calculate LSN */
replica_lsn = ((uint64) lsn_hi) << 32 | lsn_lo;
PQclear(res);
}
/* target lsn was replicated */
if (replica_lsn >= lsn)
break;
sleep(1);
if (interrupted)
elog(ERROR, "Interrupted during waiting for target LSN");
try_count++;
/* Inform user if target lsn is absent in first attempt */
if (try_count == 1)
elog(INFO, "Wait for target LSN %X/%X to be received by replica",
(uint32) (lsn >> 32), (uint32) lsn);
if (instance_config.replica_timeout > 0 &&
try_count > instance_config.replica_timeout)
elog(ERROR, "Target LSN %X/%X could not be recevied by replica "
"in %d seconds",
(uint32) (lsn >> 32), (uint32) lsn,
instance_config.replica_timeout);
}
}
/*
* Notify end of backup to PostgreSQL server.
*/
static void
pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn)
pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
PGNodeInfo *nodeInfo)
{
PGconn *conn;
PGresult *res;
@@ -1578,27 +1527,22 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn)
PQclear(res);
/* Create restore point
* only if it`s backup from master, or exclusive replica(wich connects to master)
* Only if backup is from master.
* For PG 9.5 create restore point only if pguser is superuser.
*/
if (backup != NULL && (!current.from_replica || (current.from_replica && exclusive_backup)))
if (backup != NULL && !current.from_replica &&
!(nodeInfo->server_version < 90600 &&
!nodeInfo->is_superuser))
{
const char *params[1];
char name[1024];
if (!current.from_replica)
snprintf(name, lengthof(name), "pg_probackup, backup_id %s",
base36enc(backup->start_time));
else
snprintf(name, lengthof(name), "pg_probackup, backup_id %s. Replica Backup",
base36enc(backup->start_time));
snprintf(name, lengthof(name), "pg_probackup, backup_id %s",
base36enc(backup->start_time));
params[0] = name;
res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)",
1, params);
/* Extract timeline and LSN from the result */
XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo);
/* Calculate LSN */
//restore_lsn = ((uint64) lsn_hi) << 32 | lsn_lo;
PQclear(res);
}
@@ -1679,7 +1623,11 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn)
while (1)
{
if (!PQconsumeInput(conn) || PQisBusy(conn))
if (!PQconsumeInput(conn))
elog(ERROR, "pg_stop backup() failed: %s",
PQerrorMessage(conn));
if (PQisBusy(conn))
{
pg_stop_backup_timeout++;
sleep(1);
@@ -1745,6 +1693,9 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn)
char *xlog_path,
stream_xlog_path[MAXPGPATH];
elog(WARNING, "Invalid stop_backup_lsn value %X/%X",
(uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn));
if (stream_wal)
{
pgBackupGetPath2(backup, stream_xlog_path,
@@ -1875,10 +1826,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn)
char *xlog_path,
stream_xlog_path[MAXPGPATH];
/* Wait for stop_lsn to be received by replica */
/* XXX Do we need this? */
// if (current.from_replica)
// wait_replica_wal_lsn(stop_backup_lsn, false);
/*
* Wait for stop_lsn to be archived or streamed.
* We wait for stop_lsn in stream mode just in case.
@@ -2143,6 +2090,9 @@ backup_files(void *arg)
elog(WARNING, "unexpected file type %d", buf.st_mode);
}
/* ssh connection to longer needed */
fio_disconnect();
/* Close connection */
if (arguments->conn_arg.conn)
pgut_disconnect(arguments->conn_arg.conn);
@@ -2560,7 +2510,7 @@ StreamLog(void *arg)
/*
* Start the replication
*/
elog(LOG, _("started streaming WAL at %X/%X (timeline %u)"),
elog(LOG, "started streaming WAL at %X/%X (timeline %u)",
(uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos,
stream_arg->starttli);
@@ -2601,13 +2551,13 @@ StreamLog(void *arg)
#endif
}
#else
if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, NULL,
(char *) stream_arg->basedir, stop_streaming,
standby_message_timeout, NULL, false, false) == false)
if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli,
NULL, (char *) stream_arg->basedir, stop_streaming,
standby_message_timeout, NULL, false, false) == false)
elog(ERROR, "Problem in receivexlog");
#endif
elog(LOG, _("finished streaming WAL at %X/%X (timeline %u)"),
elog(LOG, "finished streaming WAL at %X/%X (timeline %u)",
(uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli);
stream_arg->ret = 0;
@@ -2775,3 +2725,62 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn)
}
}
}
/*
* Run IDENTIFY_SYSTEM through a given connection and
* check system identifier and timeline are matching
*/
void
IdentifySystem(StreamThreadArg *stream_thread_arg)
{
PGresult *res;
uint64 stream_conn_sysidentifier = 0;
char *stream_conn_sysidentifier_str;
TimeLineID stream_conn_tli = 0;
if (!CheckServerVersionForStreaming(stream_thread_arg->conn))
{
PQfinish(stream_thread_arg->conn);
/*
* Error message already written in CheckServerVersionForStreaming().
* There's no hope of recovering from a version mismatch, so don't
* retry.
*/
elog(ERROR, "Cannot continue backup because stream connect has failed.");
}
/*
* Identify server, obtain server system identifier and timeline
*/
res = pgut_execute(stream_thread_arg->conn, "IDENTIFY_SYSTEM", 0, NULL);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
elog(WARNING,"Could not send replication command \"%s\": %s",
"IDENTIFY_SYSTEM", PQerrorMessage(stream_thread_arg->conn));
PQfinish(stream_thread_arg->conn);
elog(ERROR, "Cannot continue backup because stream connect has failed.");
}
stream_conn_sysidentifier_str = PQgetvalue(res, 0, 0);
stream_conn_tli = atoi(PQgetvalue(res, 0, 1));
/* Additional sanity, primary for PG 9.5,
* where system id can be obtained only via "IDENTIFY SYSTEM"
*/
if (!parse_uint64(stream_conn_sysidentifier_str, &stream_conn_sysidentifier, 0))
elog(ERROR, "%s is not system_identifier", stream_conn_sysidentifier_str);
if (stream_conn_sysidentifier != instance_config.system_identifier)
elog(ERROR, "System identifier mismatch. Connected PostgreSQL instance has system id: "
"" UINT64_FORMAT ". Expected: " UINT64_FORMAT ".",
stream_conn_sysidentifier, instance_config.system_identifier);
if (stream_conn_tli != current.tli)
elog(ERROR, "Timeline identifier mismatch. "
"Connected PostgreSQL instance has timeline id: %X. Expected: %X.",
stream_conn_tli, current.tli);
PQclear(res);
}
src/catalog.c
+88 -6
View File
@@ -441,22 +441,88 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx)
}
/*
* Find the last completed backup on given timeline
* Find the latest valid child of latest valid FULL backup on given timeline
*/
pgBackup *
catalog_get_last_data_backup(parray *backup_list, TimeLineID tli)
catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time)
{
int i;
pgBackup *backup = NULL;
pgBackup *full_backup = NULL;
pgBackup *tmp_backup = NULL;
char *invalid_backup_id;
/* backup_list is sorted in order of descending ID */
for (i = 0; i < parray_num(backup_list); i++)
{
backup = (pgBackup *) parray_get(backup_list, (size_t) i);
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
if ((backup->backup_mode == BACKUP_MODE_FULL &&
(backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE)) && backup->tli == tli)
{
full_backup = backup;
break;
}
}
/* Failed to find valid FULL backup to fulfill ancestor role */
if (!full_backup)
return NULL;
elog(LOG, "Latest valid FULL backup: %s",
base36enc(full_backup->start_time));
/* FULL backup is found, lets find his latest child */
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
/* only valid descendants are acceptable for evaluation */
if ((backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE) && backup->tli == tli)
return backup;
backup->status == BACKUP_STATUS_DONE))
{
switch (scan_parent_chain(backup, &tmp_backup))
{
/* broken chain */
case 0:
invalid_backup_id = base36enc_dup(tmp_backup->parent_backup);
elog(WARNING, "Backup %s has missing parent: %s. Cannot be a parent",
base36enc(backup->start_time), invalid_backup_id);
pg_free(invalid_backup_id);
continue;
/* chain is intact, but at least one parent is invalid */
case 1:
invalid_backup_id = base36enc_dup(tmp_backup->start_time);
elog(WARNING, "Backup %s has invalid parent: %s. Cannot be a parent",
base36enc(backup->start_time), invalid_backup_id);
pg_free(invalid_backup_id);
continue;
/* chain is ok */
case 2:
/* Yes, we could call is_parent() earlier - after choosing the ancestor,
* but this way we have an opportunity to detect and report all possible
* anomalies.
*/
if (is_parent(full_backup->start_time, backup, true))
{
elog(INFO, "Parent backup: %s",
base36enc(backup->start_time));
return backup;
}
}
}
/* skip yourself */
else if (backup->start_time == current_start_time)
continue;
else
{
elog(WARNING, "Backup %s has status: %s. Cannot be a parent.",
base36enc(backup->start_time), status2str(backup->status));
}
}
return NULL;
@@ -987,6 +1053,22 @@ deparse_compress_alg(int alg)
return NULL;
}
/*
* Fill PGNodeInfo struct with default values.
*/
void
pgNodeInit(PGNodeInfo *node)
{
node->block_size = 0;
node->wal_block_size = 0;
node->checksum_version = 0;
node->is_superuser = false;
node->server_version = 0;
node->server_version_str[0] = '\0';
}
/*
* Fill pgBackup struct with default values.
*/
src/checkdb.c
+41 -22
View File
@@ -79,6 +79,7 @@ typedef struct pg_indexEntry
{
Oid indexrelid;
char *name;
char *namespace;
bool heapallindexed_is_supported;
/* schema where amcheck extention is located */
char *amcheck_nspname;
@@ -98,6 +99,8 @@ pg_indexEntry_free(void *index)
if (index_ptr->name)
free(index_ptr->name);
if (index_ptr->name)
free(index_ptr->namespace);
if (index_ptr->amcheck_nspname)
free(index_ptr->amcheck_nspname);
@@ -324,7 +327,7 @@ check_indexes(void *arg)
if (progress)
elog(INFO, "Thread [%d]. Progress: (%d/%d). Amchecking index '%s.%s'",
arguments->thread_num, i + 1, n_indexes,
ind->amcheck_nspname, ind->name);
ind->namespace, ind->name);
if (arguments->conn_arg.conn == NULL)
{
@@ -362,7 +365,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck,
PGconn *db_conn)
{
PGresult *res;
char *nspname = NULL;
char *amcheck_nspname = NULL;
int i;
bool heapallindexed_is_supported = false;
parray *index_list = NULL;
@@ -391,8 +394,8 @@ get_index_list(const char *dbname, bool first_db_with_amcheck,
return NULL;
}
nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1);
strcpy(nspname, PQgetvalue(res, 0, 1));
amcheck_nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1);
strcpy(amcheck_nspname, PQgetvalue(res, 0, 1));
/* heapallindexed_is_supported is database specific */
if (strcmp(PQgetvalue(res, 0, 2), "1.0") != 0 &&
@@ -419,24 +422,28 @@ get_index_list(const char *dbname, bool first_db_with_amcheck,
if (first_db_with_amcheck)
{
res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname "
"FROM pg_index idx "
"JOIN pg_class cls ON idx.indexrelid=cls.oid "
"JOIN pg_am am ON cls.relam=am.oid "
"WHERE am.amname='btree' AND cls.relpersistence != 't'",
res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname, nmspc.nspname "
"FROM pg_catalog.pg_index idx "
"LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid "
"LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid "
"LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid "
"WHERE am.amname='btree' AND cls.relpersistence != 't' "
"ORDER BY nmspc.nspname DESC",
0, NULL);
}
else
{
res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname "
"FROM pg_index idx "
"JOIN pg_class cls ON idx.indexrelid=cls.oid "
"JOIN pg_am am ON cls.relam=am.oid "
"LEFT JOIN pg_tablespace tbl "
"ON cls.reltablespace=tbl.oid "
"AND tbl.spcname <> 'pg_global' "
"WHERE am.amname='btree' AND cls.relpersistence != 't'",
res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname, nmspc.nspname "
"FROM pg_catalog.pg_index idx "
"LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid "
"LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid "
"LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid "
"WHERE am.amname='btree' AND cls.relpersistence != 't' AND "
"(cls.reltablespace IN "
"(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') "
"OR cls.reltablespace = 0) "
"ORDER BY nmspc.nspname DESC",
0, NULL);
}
@@ -445,15 +452,24 @@ get_index_list(const char *dbname, bool first_db_with_amcheck,
{
pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry));
char *name = NULL;
char *namespace = NULL;
/* index oid */
ind->indexrelid = atoi(PQgetvalue(res, i, 0));
/* index relname */
name = PQgetvalue(res, i, 1);
ind->name = pgut_malloc(strlen(name) + 1);
strcpy(ind->name, name); /* enough buffer size guaranteed */
/* index namespace */
namespace = PQgetvalue(res, i, 2);
ind->namespace = pgut_malloc(strlen(namespace) + 1);
strcpy(ind->namespace, namespace); /* enough buffer size guaranteed */
ind->heapallindexed_is_supported = heapallindexed_is_supported;
ind->amcheck_nspname = pgut_malloc(strlen(nspname) + 1);
strcpy(ind->amcheck_nspname, nspname);
ind->amcheck_nspname = pgut_malloc(strlen(amcheck_nspname) + 1);
strcpy(ind->amcheck_nspname, amcheck_nspname);
pg_atomic_clear_flag(&ind->lock);
if (index_list == NULL)
@@ -509,7 +525,7 @@ amcheck_one_index(check_indexes_arg *arguments,
{
elog(WARNING, "Thread [%d]. Amcheck failed in database '%s' for index: '%s.%s': %s",
arguments->thread_num, arguments->conn_opt.pgdatabase,
ind->amcheck_nspname, ind->name, PQresultErrorMessage(res));
ind->namespace, ind->name, PQresultErrorMessage(res));
pfree(params[0]);
pfree(query);
@@ -519,7 +535,7 @@ amcheck_one_index(check_indexes_arg *arguments,
else
elog(LOG, "Thread [%d]. Amcheck succeeded in database '%s' for index: '%s.%s'",
arguments->thread_num,
arguments->conn_opt.pgdatabase, ind->amcheck_nspname, ind->name);
arguments->conn_opt.pgdatabase, ind->namespace, ind->name);
pfree(params[0]);
pfree(query);
@@ -633,7 +649,7 @@ do_amcheck(ConnectionOptions conn_opt, PGconn *conn)
if (check_isok)
elog(INFO, "Amcheck succeeded for database '%s'", dbname);
else
elog(WARNING, "Amcheck failed for database %s", dbname);
elog(WARNING, "Amcheck failed for database '%s'", dbname);
parray_walk(index_list, pg_indexEntry_free);
parray_free(index_list);
@@ -674,6 +690,9 @@ do_checkdb(bool need_amcheck,
PGNodeInfo nodeInfo;
PGconn *cur_conn;
/* Initialize PGInfonode */
pgNodeInit(&nodeInfo);
if (skip_block_validation && !need_amcheck)
elog(ERROR, "Option '--skip-block-validation' must be used with '--amcheck' option");
src/data.c
+5 -11
View File
@@ -356,7 +356,7 @@ prepare_page(ConnectionArgs *arguments,
((strict && !is_ptrack_support) || !strict))
{
/* show this message for checkdb or backup without ptrack support */
elog(WARNING, "CORRUPTION in file %s, block %u",
elog(WARNING, "Corruption detected in file \"%s\", block %u",
file->path, blknum);
}
@@ -585,10 +585,7 @@ backup_data_file(backup_files_arg* arguments,
}
if (file->size % BLCKSZ != 0)
{
fio_fclose(in);
elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size);
}
elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size);
/*
* Compute expected number of blocks in the file.
@@ -625,7 +622,7 @@ backup_data_file(backup_files_arg* arguments,
if (rc == PAGE_CHECKSUM_MISMATCH && is_ptrack_support)
goto RetryUsingPtrack;
if (rc < 0)
elog(ERROR, "Failed to read file %s: %s",
elog(ERROR, "Failed to read file \"%s\": %s",
file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc));
n_blocks_read = rc;
}
@@ -1212,10 +1209,7 @@ check_data_file(ConnectionArgs *arguments,
}
if (file->size % BLCKSZ != 0)
{
fclose(in);
elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size);
}
elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size);
/*
* Compute expected number of blocks in the file.
@@ -1400,7 +1394,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (crc != file->crc)
{
elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X",
file->path, file->crc, crc);
file->path, crc, file->crc);
is_valid = false;
}
src/delete.c
+64 -13
View File
@@ -175,6 +175,8 @@ int do_retention(void)
if (delete_wal && !dry_run)
do_retention_wal();
/* TODO: consider dry-run flag */
if (!backup_merged)
elog(INFO, "There are no backups to merge by retention policy");
@@ -203,13 +205,15 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
int i;
time_t current_time;
parray *redundancy_full_backup_list = NULL;
/* For retention calculation */
uint32 n_full_backups = 0;
int cur_full_backup_num = 0;
time_t days_threshold = 0;
/* For fancy reporting */
float actual_window = 0;
uint32 actual_window = 0;
/* Get current time */
current_time = time(NULL);
@@ -221,15 +225,27 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
/* Consider only valid backups for Redundancy */
/* Consider only valid FULL backups for Redundancy */
if (instance_config.retention_redundancy > 0 &&
backup->backup_mode == BACKUP_MODE_FULL &&
(backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE))
{
n_full_backups++;
/* Add every FULL backup that satisfy Redundancy policy to separate list */
if (n_full_backups <= instance_config.retention_redundancy)
{
if (!redundancy_full_backup_list)
redundancy_full_backup_list = parray_new();
parray_append(redundancy_full_backup_list, backup);
}
}
}
/* Sort list of full backups to keep */
if (redundancy_full_backup_list)
parray_qsort(redundancy_full_backup_list, pgBackupCompareIdDesc);
}
if (instance_config.retention_window > 0)
@@ -242,8 +258,20 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
for (i = (int) parray_num(backup_list) - 1; i >= 0; i--)
{
bool redundancy_keep = false;
pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i);
/* check if backup`s FULL ancestor is in redundancy list */
if (redundancy_full_backup_list)
{
pgBackup *full_backup = find_parent_full_backup(backup);
if (full_backup && parray_bsearch(redundancy_full_backup_list,
full_backup,
pgBackupCompareIdDesc))
redundancy_keep = true;
}
/* Remember the serial number of latest valid FULL backup */
if (backup->backup_mode == BACKUP_MODE_FULL &&
(backup->status == BACKUP_STATUS_OK ||
@@ -252,9 +280,11 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
cur_full_backup_num++;
}
/* Check if backup in needed by retention policy */
/* Check if backup in needed by retention policy
* TODO: consider that ERROR backup most likely to have recovery_time == 0
*/
if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) &&
(instance_config.retention_redundancy <= (n_full_backups - cur_full_backup_num)))
(instance_config.retention_redundancy == 0 || !redundancy_keep))
{
/* This backup is not guarded by retention
*
@@ -324,7 +354,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
}
/* Message about retention state of backups
* TODO: Float is ugly, rewrite somehow.
* TODO: message is ugly, rewrite it to something like show table in stdout.
*/
cur_full_backup_num = 1;
@@ -340,9 +370,10 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
if (backup->recovery_time == 0)
actual_window = 0;
else
actual_window = ((float)current_time - (float)backup->recovery_time)/(60 * 60 * 24);
actual_window = (current_time - backup->recovery_time)/(60 * 60 * 24);
elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %.2fd/%ud. %s",
/* TODO: add ancestor(chain full backup) ID */
elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s",
base36enc(backup->start_time),
pgBackupGetBackupMode(backup),
status2str(backup->status),
@@ -801,10 +832,13 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
int
do_delete_instance(void)
{
parray *backup_list;
int i;
parray *backup_list;
parray *xlog_files_list;
int i;
int rc;
char instance_config_path[MAXPGPATH];
/* Delete all backups. */
backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
@@ -821,23 +855,40 @@ do_delete_instance(void)
parray_free(backup_list);
/* Delete all wal files. */
delete_walfiles(InvalidXLogRecPtr, 0, instance_config.xlog_seg_size);
xlog_files_list = parray_new();
dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
for (i = 0; i < parray_num(xlog_files_list); i++)
{
pgFile *wal_file = (pgFile *) parray_get(xlog_files_list, i);
if (S_ISREG(wal_file->mode))
{
rc = unlink(wal_file->path);
if (rc != 0)
elog(WARNING, "Failed to remove file \"%s\": %s",
wal_file->path, strerror(errno));
}
}
/* Cleanup */
parray_walk(xlog_files_list, pgFileFree);
parray_free(xlog_files_list);
/* Delete backup instance config file */
join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
if (remove(instance_config_path))
{
elog(ERROR, "can't remove \"%s\": %s", instance_config_path,
elog(ERROR, "Can't remove \"%s\": %s", instance_config_path,
strerror(errno));
}
/* Delete instance root directories */
if (rmdir(backup_instance_path) != 0)
elog(ERROR, "can't remove \"%s\": %s", backup_instance_path,
elog(ERROR, "Can't remove \"%s\": %s", backup_instance_path,
strerror(errno));
if (rmdir(arclog_path) != 0)
elog(ERROR, "can't remove \"%s\": %s", arclog_path,
elog(ERROR, "Can't remove \"%s\": %s", arclog_path,
strerror(errno));
elog(INFO, "Instance '%s' successfully deleted", instance_name);
src/dir.c
+10 -10
View File
@@ -122,7 +122,7 @@ static int BlackListCompare(const void *str1, const void *str2);
static char dir_check_file(pgFile *file);
static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
bool omit_symlink, parray *black_list,
bool follow_symlink, parray *black_list,
int external_dir_num, fio_location location);
static void opt_path_map(ConfigOption *opt, const char *arg,
TablespaceList *list, const char *type);
@@ -159,14 +159,14 @@ dir_create_dir(const char *dir, mode_t mode)
}
pgFile *
pgFileNew(const char *path, const char *rel_path, bool omit_symlink,
pgFileNew(const char *path, const char *rel_path, bool follow_symlink,
int external_dir_num, fio_location location)
{
struct stat st;
pgFile *file;
/* stat the file */
if (fio_stat(path, &st, omit_symlink, location) < 0)
if (fio_stat(path, &st, follow_symlink, location) < 0)
{
/* file not found is not an error case */
if (errno == ENOENT)
@@ -445,11 +445,11 @@ BlackListCompare(const void *str1, const void *str2)
* List files, symbolic links and directories in the directory "root" and add
* pgFile objects to "files". We add "root" to "files" if add_root is true.
*
* When omit_symlink is true, symbolic link is ignored and only file or
* When follow_symlink is true, symbolic link is ignored and only file or
* directory linked to will be listed.
*/
void
dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink,
bool add_root, int external_dir_num, fio_location location)
{
pgFile *file;
@@ -490,7 +490,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
parray_qsort(black_list, BlackListCompare);
}
file = pgFileNew(root, "", omit_symlink, external_dir_num, location);
file = pgFileNew(root, "", follow_symlink, external_dir_num, location);
if (file == NULL)
{
/* For external directory this is not ok */
@@ -512,7 +512,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
if (add_root)
parray_append(files, file);
dir_list_file_internal(files, file, exclude, omit_symlink, black_list,
dir_list_file_internal(files, file, exclude, follow_symlink, black_list,
external_dir_num, location);
if (!add_root)
@@ -731,7 +731,7 @@ dir_check_file(pgFile *file)
*/
static void
dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
bool omit_symlink, parray *black_list,
bool follow_symlink, parray *black_list,
int external_dir_num, fio_location location)
{
DIR *dir;
@@ -764,7 +764,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
join_path_components(child, parent->path, dent->d_name);
join_path_components(rel_child, parent->rel_path, dent->d_name);
file = pgFileNew(child, rel_child, omit_symlink, external_dir_num,
file = pgFileNew(child, rel_child, follow_symlink, external_dir_num,
location);
if (file == NULL)
continue;
@@ -821,7 +821,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
* recursively.
*/
if (S_ISDIR(file->mode))
dir_list_file_internal(files, file, exclude, omit_symlink,
dir_list_file_internal(files, file, exclude, follow_symlink,
black_list, external_dir_num, location);
}
src/help.c
+13
View File
@@ -97,9 +97,11 @@ help_pg_probackup(void)
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
printf(_(" [--ssh-options]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" [--format=format]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" [-D pgdata-path] [-C]\n"));
@@ -126,6 +128,7 @@ help_pg_probackup(void)
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
printf(_(" [--ssh-options]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n"));
@@ -143,6 +146,7 @@ help_pg_probackup(void)
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
printf(_(" [--ssh-options]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME);
printf(_(" [-i backup-id] [--progress] [-j num-threads]\n"));
@@ -151,22 +155,27 @@ help_pg_probackup(void)
printf(_(" [--recovery-target-timeline=timeline]\n"));
printf(_(" [--recovery-target-name=target-name]\n"));
printf(_(" [--skip-block-validation]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME);
printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n"));
printf(_(" [--amcheck] [--skip-block-validation]\n"));
printf(_(" [--heapallindexed]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME);
printf(_(" [--instance=instance_name [-i backup-id]]\n"));
printf(_(" [--format=format]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" [--wal] [-i backup-id | --expired | --merge-expired]\n"));
printf(_(" [--dry-run]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" -i backup-id [--progress] [-j num-threads]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME);
printf(_(" --instance=instance_name\n"));
@@ -174,9 +183,11 @@ help_pg_probackup(void)
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
printf(_(" [--ssh-options]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME);
printf(_(" --instance=instance_name\n"));
printf(_(" [--help]\n"));
printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" --wal-file-path=wal-file-path\n"));
@@ -188,6 +199,7 @@ help_pg_probackup(void)
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
printf(_(" [--ssh-options]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" --wal-file-path=wal-file-path\n"));
@@ -195,6 +207,7 @@ help_pg_probackup(void)
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
printf(_(" [--ssh-options]\n"));
printf(_(" [--help]\n"));
if ((PROGRAM_URL || PROGRAM_EMAIL))
{
src/parsexlog.c
+14
View File
@@ -510,9 +510,18 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn,
xlogreader = InitXLogPageRead(&reader_data, archivedir, target_tli,
wal_seg_size, false, false, true);
if (xlogreader == NULL)
elog(ERROR, "Out of memory");
xlogreader->system_identifier = instance_config.system_identifier;
res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL;
/* Didn't find 'target_lsn' and there is no error, return false */
if (errormsg)
elog(WARNING, "Could not read WAL record at %X/%X: %s",
(uint32) (target_lsn >> 32), (uint32) (target_lsn), errormsg);
CleanupXLogPageRead(xlogreader);
XLogReaderFree(xlogreader);
@@ -551,6 +560,11 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn,
xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size,
false, false, true);
if (xlogreader == NULL)
elog(ERROR, "Out of memory");
xlogreader->system_identifier = instance_config.system_identifier;
/*
* Calculate startpoint. Decide: we should use 'start_lsn' or offset 0.
*/
src/pg_probackup.c
+5 -12
View File
@@ -20,7 +20,9 @@
#include "utils/thread.h"
#include <time.h>
const char *PROGRAM_NAME = NULL;
const char *PROGRAM_NAME = NULL; /* PROGRAM_NAME_FULL without .exe suffix
* if any */
const char *PROGRAM_NAME_FULL = NULL;
const char *PROGRAM_FULL_PATH = NULL;
const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup";
const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues";
@@ -45,8 +47,6 @@ typedef enum ProbackupSubcmd
} ProbackupSubcmd;
char *pg_probackup; /* Program name (argv[0]) */
/* directory options */
char *backup_path = NULL;
/*
@@ -235,7 +235,7 @@ main(int argc, char *argv[])
struct stat stat_buf;
int rc;
pg_probackup = argv[0];
PROGRAM_NAME_FULL = argv[0];
/* Initialize current backup */
pgBackupInit(&current);
@@ -612,17 +612,10 @@ main(int argc, char *argv[])
return do_init();
case BACKUP_CMD:
{
const char *backup_mode;
time_t start_time;
time_t start_time = time(NULL);
start_time = time(NULL);
backup_mode = deparse_backup_mode(current.backup_mode);
current.stream = stream_wal;
elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote %s",
PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name,
stream_wal ? "true" : "false", instance_config.remote.host ? "true" : "false");
/* sanity */
if (current.backup_mode == BACKUP_MODE_INVALID)
elog(ERROR, "required parameter not specified: BACKUP_MODE "
src/pg_probackup.h
+16 -10
View File
@@ -36,6 +36,7 @@
/* pgut client variables and full path */
extern const char *PROGRAM_NAME;
extern const char *PROGRAM_NAME_FULL;
extern const char *PROGRAM_FULL_PATH;
extern const char *PROGRAM_URL;
extern const char *PROGRAM_EMAIL;
@@ -187,8 +188,8 @@ typedef enum ShowFormat
#define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */
#define FILE_NOT_FOUND (-2) /* file disappeared during backup */
#define BLOCKNUM_INVALID (-1)
#define PROGRAM_VERSION "2.1.3"
#define AGENT_PROTOCOL_VERSION 20103
#define PROGRAM_VERSION "2.1.4"
#define AGENT_PROTOCOL_VERSION 20104
typedef struct ConnectionOptions
@@ -247,9 +248,11 @@ typedef struct PGNodeInfo
uint32 block_size;
uint32 wal_block_size;
uint32 checksum_version;
bool is_superuser;
int server_version;
char server_version_str[100];
char program_version[100];
char server_version[100];
} PGNodeInfo;
typedef struct pgBackup pgBackup;
@@ -289,7 +292,6 @@ struct pgBackup
int compress_level;
/* Fields needed for compatibility check */
PGNodeInfo nodeInfo;
uint32 block_size;
uint32 wal_block_size;
uint32 checksum_version;
@@ -412,7 +414,6 @@ typedef struct BackupPageHeader
#define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0)
/* directory options */
extern char *pg_probackup;
extern char *backup_path;
extern char backup_instance_path[MAXPGPATH];
extern char arclog_path[MAXPGPATH];
@@ -472,7 +473,7 @@ extern const char *pgdata_exclude_dir[];
/* in backup.c */
extern int do_backup(time_t start_time, bool no_validate);
extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt,
extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt,
char *pgdata);
extern BackupMode parse_backup_mode(const char *value);
extern const char *deparse_backup_mode(BackupMode mode);
@@ -553,7 +554,8 @@ extern parray *catalog_get_backup_list(time_t requested_backup_id);
extern void catalog_lock_backup_list(parray *backup_list, int from_idx,
int to_idx);
extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
TimeLineID tli);
TimeLineID tli,
time_t current_start_time);
extern void pgBackupWriteControl(FILE *out, pgBackup *backup);
extern void write_backup_filelist(pgBackup *backup, parray *files,
const char *root, parray *external_list);
@@ -563,6 +565,7 @@ extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len,
extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len,
const char *subdir1, const char *subdir2);
extern int pgBackupCreateDir(pgBackup *backup);
extern void pgNodeInit(PGNodeInfo *node);
extern void pgBackupInit(pgBackup *backup);
extern void pgBackupFree(void *backup);
extern int pgBackupCompareId(const void *f1, const void *f2);
@@ -577,6 +580,7 @@ extern bool in_backup_list(parray *backup_list, pgBackup *target_backup);
extern int get_backup_index_number(parray *backup_list, pgBackup *backup);
extern bool launch_agent(void);
extern void launch_ssh(char* argv[]);
extern void wait_ssh(void);
#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS
#define COMPRESS_LEVEL_DEFAULT 1
@@ -586,7 +590,7 @@ extern const char* deparse_compress_alg(int alg);
/* in dir.c */
extern void dir_list_file(parray *files, const char *root, bool exclude,
bool omit_symlink, bool add_root, int external_dir_num, fio_location location);
bool follow_symlink, bool add_root, int external_dir_num, fio_location location);
extern void create_data_directories(parray *dest_files,
const char *data_dir,
@@ -619,7 +623,7 @@ extern bool fileExists(const char *path, fio_location location);
extern size_t pgFileSize(const char *path);
extern pgFile *pgFileNew(const char *path, const char *rel_path,
bool omit_symlink, int external_dir_num,
bool follow_symlink, int external_dir_num,
fio_location location);
extern pgFile *pgFileInit(const char *path, const char *rel_path);
extern void pgFileDelete(pgFile *file);
@@ -695,6 +699,8 @@ extern bool parse_page(Page page, XLogRecPtr *lsn);
int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size,
CompressAlg alg, int level, const char **errormsg);
extern void pretty_size(int64 size, char *buf, size_t len);
extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo);
extern void check_system_identifiers(PGconn *conn, char *pgdata);
src/restore.c
+13 -3
View File
@@ -160,7 +160,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
if (!satisfy_recovery_target(current_backup, rt))
{
if (target_backup_id != INVALID_BACKUP_ID)
elog(ERROR, "target backup %s does not satisfy restore options",
elog(ERROR, "Requested backup %s does not satisfy restore options",
base36enc(target_backup_id));
else
/* Try to find another backup that satisfies target options */
@@ -175,8 +175,16 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
}
}
/* TODO: Show latest possible target */
if (dest_backup == NULL)
elog(ERROR, "Backup satisfying target options is not found.");
{
/* Failed to find target backup */
if (target_backup_id)
elog(ERROR, "Requested backup %s is not found.", base36enc(target_backup_id));
else
elog(ERROR, "Backup satisfying target options is not found.");
/* TODO: check if user asked PITR or just restore of latest backup */
}
/* If we already found dest_backup, look for full backup. */
if (dest_backup->backup_mode == BACKUP_MODE_FULL)
@@ -789,7 +797,7 @@ create_recovery_conf(time_t backup_id,
elog(LOG, "creating recovery.conf");
snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata);
fp = fio_fopen(path, "wt", FIO_DB_HOST);
fp = fio_fopen(path, "w", FIO_DB_HOST);
if (fp == NULL)
elog(ERROR, "cannot open recovery.conf \"%s\": %s", path,
strerror(errno));
@@ -954,6 +962,7 @@ read_timeline_history(TimeLineID targetTLI)
return result;
}
/* TODO: do not ignore timelines. What if requested target located in different timeline? */
bool
satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt)
{
@@ -969,6 +978,7 @@ satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt)
return true;
}
/* TODO description */
bool
satisfy_timeline(const parray *timelines, const pgBackup *backup)
{
src/show.c
+1 -1
View File
@@ -118,7 +118,7 @@ do_show(time_t requested_backup_id)
return show_backup(requested_backup_id);
}
static void
void
pretty_size(int64 size, char *buf, size_t len)
{
int exp = 0;
src/util.c
+8 -14
View File
@@ -153,7 +153,7 @@ get_current_timeline(bool safe)
size_t size;
/* First fetch file... */
buffer = slurpFile(instance_config.pgdata, "global/pg_control", &size,
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size,
safe, FIO_DB_HOST);
if (safe && buffer == NULL)
return 0;
@@ -196,7 +196,7 @@ get_checkpoint_location(PGconn *conn)
size_t size;
ControlFileData ControlFile;
buffer = fetchFile(conn, "global/pg_control", &size);
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
@@ -212,7 +212,7 @@ get_system_identifier(const char *pgdata_path)
size_t size;
/* First fetch file... */
buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_DB_HOST);
buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
if (buffer == NULL)
return 0;
digestControlFile(&ControlFile, buffer, size);
@@ -246,7 +246,7 @@ get_remote_system_identifier(PGconn *conn)
size_t size;
ControlFileData ControlFile;
buffer = fetchFile(conn, "global/pg_control", &size);
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
@@ -263,9 +263,7 @@ get_xlog_seg_size(char *pgdata_path)
size_t size;
/* First fetch file... */
buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_DB_HOST);
if (buffer == NULL)
return 0;
buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
@@ -283,7 +281,7 @@ get_data_checksum_version(bool safe)
size_t size;
/* First fetch file... */
buffer = slurpFile(instance_config.pgdata, "global/pg_control", &size,
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size,
safe, FIO_DB_HOST);
if (buffer == NULL)
return 0;
@@ -301,9 +299,8 @@ get_pgcontrol_checksum(const char *pgdata_path)
size_t size;
/* First fetch file... */
buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_BACKUP_HOST);
if (buffer == NULL)
return 0;
buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_BACKUP_HOST);
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
@@ -325,9 +322,6 @@ set_min_recovery_point(pgFile *file, const char *backup_path,
/* First fetch file content */
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
if (buffer == NULL)
elog(ERROR, "ERROR");
digestControlFile(&ControlFile, buffer, size);
elog(LOG, "Current minRecPoint %X/%X",
src/utils/file.c
+44 -11
View File
@@ -333,6 +333,21 @@ int fio_open(char const* path, int mode, fio_location location)
return fd;
}
/* Close ssh session */
void
fio_disconnect(void)
{
if (fio_stdin)
{
SYS_CHECK(close(fio_stdin));
SYS_CHECK(close(fio_stdout));
fio_stdin = 0;
fio_stdout = 0;
wait_ssh();
}
}
/* Open stdio file */
FILE* fio_fopen(char const* path, char const* mode, fio_location location)
{
@@ -340,14 +355,30 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location)
if (fio_is_remote(location))
{
int flags = O_RDWR|O_CREAT;
int flags = 0;
int fd;
if (strcmp(mode, PG_BINARY_W) == 0) {
flags |= O_TRUNC|PG_BINARY;
} else if (strncmp(mode, PG_BINARY_R, strlen(PG_BINARY_R)) == 0) {
flags |= PG_BINARY;
flags = O_TRUNC|PG_BINARY|O_RDWR|O_CREAT;
} else if (strcmp(mode, "w") == 0) {
flags = O_TRUNC|O_RDWR|O_CREAT;
} else if (strcmp(mode, PG_BINARY_R) == 0) {
flags = O_RDONLY|PG_BINARY;
} else if (strcmp(mode, "r") == 0) {
flags = O_RDONLY;
} else if (strcmp(mode, PG_BINARY_R "+") == 0) {
/* stdio fopen("rb+") actually doesn't create unexisted file, but probackup frequently
* needs to open existed file or create new one if not exists.
* In stdio it can be done using two fopen calls: fopen("r+") and if failed then fopen("w").
* But to eliminate extra call which especially critical in case of remote connection
* we change r+ semantic to create file if not exists.
*/
flags = O_RDWR|O_CREAT|PG_BINARY;
} else if (strcmp(mode, "r+") == 0) { /* see comment above */
flags |= O_RDWR|O_CREAT;
} else if (strcmp(mode, "a") == 0) {
flags |= O_APPEND;
flags |= O_CREAT|O_RDWR|O_APPEND;
} else {
Assert(false);
}
fd = fio_open(path, flags, location);
if (fd >= 0)
@@ -632,7 +663,7 @@ int fio_fstat(int fd, struct stat* st)
}
/* Get information about file */
int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location)
int fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location)
{
if (fio_is_remote(location))
{
@@ -641,7 +672,7 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_locati
hdr.cop = FIO_STAT;
hdr.handle = -1;
hdr.arg = follow_symlinks;
hdr.arg = follow_symlink;
hdr.size = path_len;
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
@@ -660,7 +691,7 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_locati
}
else
{
return follow_symlinks ? stat(path, st) : lstat(path, st);
return follow_symlink ? stat(path, st) : lstat(path, st);
}
}
@@ -1148,8 +1179,10 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file,
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
Assert(hdr.cop == FIO_PAGE);
if (hdr.arg < 0) /* read error */
return hdr.arg;
if ((int)hdr.arg < 0) /* read error */
{
return (int)hdr.arg;
}
blknum = hdr.arg;
if (hdr.size == 0) /* end of segment */
@@ -1205,7 +1238,7 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req)
{
hdr.arg = -errno;
hdr.size = 0;
Assert(hdr.arg < 0);
Assert((int)hdr.arg < 0);
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
}
else
src/utils/file.h
+1
View File
@@ -90,6 +90,7 @@ extern int fio_seek(int fd, off_t offs);
extern int fio_fstat(int fd, struct stat* st);
extern int fio_truncate(int fd, off_t size);
extern int fio_close(int fd);
extern void fio_disconnect(void);
extern int fio_rename(char const* old_path, char const* new_path, fio_location location);
extern int fio_symlink(char const* target, char const* link_path, fio_location location);
src/utils/pgut.c
+4 -1
View File
@@ -206,6 +206,9 @@ pgut_get_conninfo_string(PGconn *conn)
return connstr;
}
/* TODO: it is better to use PQconnectdbParams like in psql
* It will allow to set application_name for pg_probackup
*/
PGconn *
pgut_connect(const char *host, const char *port,
const char *dbname, const char *username)
@@ -419,7 +422,7 @@ pgut_execute_parallel(PGconn* conn,
}
if (!PQconsumeInput(conn))
elog(ERROR, "query failed: %squery was: %s",
elog(ERROR, "query failed: %s query was: %s",
PQerrorMessage(conn), query);
/* query is no done */
src/utils/remote.c
+41 -13
View File
@@ -5,6 +5,12 @@
#include <sys/wait.h>
#include <signal.h>
#ifdef WIN32
#define __thread __declspec(thread)
#else
#include <pthread.h>
#endif
#include "pg_probackup.h"
#include "file.h"
@@ -52,7 +58,8 @@ static int split_options(int argc, char* argv[], int max_options, char* options)
return argc;
}
static int child_pid;
static __thread int child_pid;
#if 0
static void kill_child(void)
{
@@ -60,6 +67,14 @@ static void kill_child(void)
}
#endif
void wait_ssh(void)
{
int status;
waitpid(child_pid, &status, 0);
elog(LOG, "SSH process %d is terminated with status %d", child_pid, status);
}
#ifdef WIN32
void launch_ssh(char* argv[])
{
@@ -76,6 +91,10 @@ void launch_ssh(char* argv[])
}
#endif
static bool needs_quotes(char const* path)
{
return strchr(path, ' ') != NULL;
}
bool launch_agent(void)
{
@@ -87,7 +106,7 @@ bool launch_agent(void)
ssh_argc = 0;
#ifdef WIN32
ssh_argv[ssh_argc++] = pg_probackup;
ssh_argv[ssh_argc++] = PROGRAM_NAME_FULL;
ssh_argv[ssh_argc++] = "ssh";
ssh_argc += 2; /* reserve space for pipe descriptors */
#endif
@@ -107,11 +126,9 @@ bool launch_agent(void)
if (instance_config.remote.ssh_options != NULL) {
ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, pg_strdup(instance_config.remote.ssh_options));
}
if (num_threads > 1)
{
ssh_argv[ssh_argc++] = "-o";
ssh_argv[ssh_argc++] = "PasswordAuthentication=no";
}
ssh_argv[ssh_argc++] = "-o";
ssh_argv[ssh_argc++] = "PasswordAuthentication=no";
ssh_argv[ssh_argc++] = "-o";
ssh_argv[ssh_argc++] = "Compression=no";
@@ -125,7 +142,7 @@ bool launch_agent(void)
if (instance_config.remote.path)
{
char const* probackup = pg_probackup;
char const* probackup = PROGRAM_NAME_FULL;
char* sep = strrchr(probackup, '/');
if (sep != NULL) {
probackup = sep + 1;
@@ -137,14 +154,25 @@ bool launch_agent(void)
probackup = sep + 1;
}
}
snprintf(cmd, sizeof(cmd), "%s\\%s agent %s",
instance_config.remote.path, probackup, PROGRAM_VERSION);
if (needs_quotes(instance_config.remote.path) || needs_quotes(PROGRAM_NAME_FULL))
snprintf(cmd, sizeof(cmd), "\"%s\\%s\" agent %s",
instance_config.remote.path, probackup, PROGRAM_VERSION);
else
snprintf(cmd, sizeof(cmd), "%s\\%s agent %s",
instance_config.remote.path, probackup, PROGRAM_VERSION);
#else
snprintf(cmd, sizeof(cmd), "%s/%s agent %s",
instance_config.remote.path, probackup, PROGRAM_VERSION);
if (needs_quotes(instance_config.remote.path) || needs_quotes(PROGRAM_NAME_FULL))
snprintf(cmd, sizeof(cmd), "\"%s/%s\" agent %s",
instance_config.remote.path, probackup, PROGRAM_VERSION);
else
snprintf(cmd, sizeof(cmd), "%s/%s agent %s",
instance_config.remote.path, probackup, PROGRAM_VERSION);
#endif
} else {
snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION);
if (needs_quotes(PROGRAM_NAME_FULL))
snprintf(cmd, sizeof(cmd), "\"%s\" agent %s", PROGRAM_NAME_FULL, PROGRAM_VERSION);
else
snprintf(cmd, sizeof(cmd), "%s agent %s", PROGRAM_NAME_FULL, PROGRAM_VERSION);
}
#ifdef WIN32
src/validate.c
+2 -2
View File
@@ -240,7 +240,7 @@ pgBackupValidateFiles(void *arg)
if (file->write_size != st.st_size)
{
elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu",
file->path, file->write_size, (unsigned long) st.st_size);
file->path, (unsigned long) st.st_size, file->write_size);
arguments->corrupted = true;
break;
}
@@ -276,7 +276,7 @@ pgBackupValidateFiles(void *arg)
if (crc != file->crc)
{
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
file->path, file->crc, crc);
file->path, crc, file->crc);
arguments->corrupted = true;
}
}
tests/Readme.md
+3
View File
@@ -29,6 +29,9 @@ Remote backup depends on key authentithication to local machine via ssh as curre
Run suit of basic simple tests:
export PG_PROBACKUP_TEST_BASIC=ON
Run ptrack tests:
export PG_PROBACKUP_PTRACK=ON
Usage:
pip install testgres
tests/__init__.py
+7 -3
View File
@@ -12,8 +12,9 @@ from . import init, merge, option, show, compatibility, \
def load_tests(loader, tests, pattern):
suite = unittest.TestSuite()
if os.environ['PG_PROBACKUP_TEST_BASIC'] == 'ON':
loader.testMethodPrefix = 'test_basic'
if 'PG_PROBACKUP_TEST_BASIC' in os.environ:
if os.environ['PG_PROBACKUP_TEST_BASIC'] == 'ON':
loader.testMethodPrefix = 'test_basic'
# suite.addTests(loader.loadTestsFromModule(auth_test))
suite.addTests(loader.loadTestsFromModule(archive))
@@ -36,7 +37,6 @@ def load_tests(loader, tests, pattern):
suite.addTests(loader.loadTestsFromModule(merge))
suite.addTests(loader.loadTestsFromModule(option))
suite.addTests(loader.loadTestsFromModule(page))
# suite.addTests(loader.loadTestsFromModule(ptrack))
suite.addTests(loader.loadTestsFromModule(pgpro560))
suite.addTests(loader.loadTestsFromModule(pgpro589))
suite.addTests(loader.loadTestsFromModule(pgpro2068))
@@ -49,6 +49,10 @@ def load_tests(loader, tests, pattern):
suite.addTests(loader.loadTestsFromModule(time_stamp))
suite.addTests(loader.loadTestsFromModule(validate))
if 'PG_PROBACKUP_PTRACK' in os.environ:
if os.environ['PG_PROBACKUP_PTRACK'] == 'ON':
suite.addTests(loader.loadTestsFromModule(ptrack))
return suite
# test_pgpro434_2 unexpected success
tests/archive.py
+290 -36
View File
@@ -25,8 +25,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s'}
)
'checkpoint_timeout': '30s'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
@@ -228,10 +228,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
@@ -242,7 +240,6 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node,
options=[
"--archive-timeout=60",
"--stream",
"--log-level-file=info"],
gdb=True)
@@ -255,21 +252,98 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
gdb.continue_execution_until_exit()
log_file = os.path.join(backup_dir, 'log/pg_probackup.log')
log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log')
with open(log_file, 'r') as f:
log_content = f.read()
self.assertNotIn(
"ERROR: pg_stop_backup doesn't answer",
log_content,
"pg_stop_backup timeouted")
self.assertIn(
"ERROR: Switched WAL segment 000000010000000000000002 "
"could not be archived in 60 seconds",
log_content)
self.assertIn(
"ERROR: Switched WAL segment 000000010000000000000002 "
"could not be archived in 60 seconds",
log_content)
log_file = os.path.join(node.logs_dir, 'postgresql.log')
with open(log_file, 'r') as f:
log_content = f.read()
self.assertNotIn(
'FailedAssertion',
log_content,
'PostgreSQL crashed because of a failed assert')
self.assertNotIn(
'FailedAssertion',
log_content,
'PostgreSQL crashed because of a failed assert')
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_pgpro434_4(self):
"""
Check pg_stop_backup_timeout, needed backup_timeout
Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
gdb = self.backup_node(
backup_dir, 'node', node,
options=[
"--archive-timeout=60",
"--log-level-file=info"],
gdb=True)
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
node.append_conf(
'postgresql.auto.conf', "archive_command = 'exit 1'")
node.reload()
os.environ["PGAPPNAME"] = "foo"
pid = node.safe_psql(
"postgres",
"SELECT pid "
"FROM pg_stat_activity "
"WHERE application_name = 'pg_probackup'").rstrip()
os.environ["PGAPPNAME"] = "pg_probackup"
postgres_gdb = self.gdb_attach(pid)
postgres_gdb.set_breakpoint('do_pg_stop_backup')
postgres_gdb.continue_execution_until_running()
gdb.continue_execution_until_exit()
# gdb._execute('detach')
log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log')
with open(log_file, 'r') as f:
log_content = f.read()
self.assertIn(
"ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it",
log_content)
log_file = os.path.join(node.logs_dir, 'postgresql.log')
with open(log_file, 'r') as f:
log_content = f.read()
self.assertNotIn(
'FailedAssertion',
log_content,
'PostgreSQL crashed because of a failed assert')
# Clean after yourself
self.del_test_dir(module_name, fname)
@@ -311,17 +385,33 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
"from generate_series(0,100500) i")
log_file = os.path.join(node.logs_dir, 'postgresql.log')
self.switch_wal_segment(node)
sleep(1)
with open(log_file, 'r') as f:
log_content = f.read()
self.assertTrue(
'LOG: archive command failed with exit code 1' in log_content and
'DETAIL: The failed archive command was:' in log_content and
'INFO: pg_probackup archive-push from' in log_content and
'ERROR: WAL segment ' in log_content and
'{0}" already exists.'.format(filename) in log_content,
'Expecting error messages about failed archive_command'
)
self.assertFalse('pg_probackup archive-push completed successfully' in log_content)
self.assertIn(
'LOG: archive command failed with exit code 1',
log_content)
self.assertIn(
'DETAIL: The failed archive command was:',
log_content)
self.assertIn(
'INFO: pg_probackup archive-push from',
log_content)
self.assertIn(
'ERROR: WAL segment ',
log_content)
self.assertIn(
'already exists.',
log_content)
self.assertNotIn(
'pg_probackup archive-push completed successfully', log_content)
if self.get_version(node) < 100000:
wal_src = os.path.join(
@@ -342,9 +432,10 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
with open(log_file, 'r') as f:
log_content = f.read()
self.assertTrue(
'pg_probackup archive-push completed successfully' in log_content,
'Expecting messages about successfull execution archive_command')
self.assertIn(
'pg_probackup archive-push completed successfully',
log_content)
# Clean after yourself
self.del_test_dir(module_name, fname)
@@ -386,16 +477,23 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
"from generate_series(0,100500) i")
log_file = os.path.join(node.logs_dir, 'postgresql.log')
self.switch_wal_segment(node)
sleep(1)
with open(log_file, 'r') as f:
log_content = f.read()
self.assertTrue(
'LOG: archive command failed with exit code 1' in log_content and
'DETAIL: The failed archive command was:' in log_content and
'INFO: pg_probackup archive-push from' in log_content and
'{0}" already exists.'.format(filename) in log_content,
'Expecting error messages about failed archive_command'
)
self.assertFalse('pg_probackup archive-push completed successfully' in log_content)
self.assertIn(
'LOG: archive command failed with exit code 1', log_content)
self.assertIn(
'DETAIL: The failed archive command was:', log_content)
self.assertIn(
'INFO: pg_probackup archive-push from', log_content)
self.assertIn(
'{0}" already exists.'.format(filename), log_content)
self.assertNotIn(
'pg_probackup archive-push completed successfully', log_content)
self.set_archiving(backup_dir, 'node', node, overwrite=True)
node.reload()
@@ -411,6 +509,162 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_archive_push_partial_file_exists(self):
"""Archive-push if stale '.partial' file exists"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# this backup is needed only for validation to xid
self.backup_node(backup_dir, 'node', node)
node.safe_psql(
"postgres",
"create table t1(a int)")
xid = node.safe_psql(
"postgres",
"INSERT INTO t1 VALUES (1) RETURNING (xmin)").rstrip()
if self.get_version(node) < 100000:
filename_orig = node.safe_psql(
"postgres",
"SELECT file_name "
"FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip()
else:
filename_orig = node.safe_psql(
"postgres",
"SELECT file_name "
"FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip()
# form up path to next .partial WAL segment
wals_dir = os.path.join(backup_dir, 'wal', 'node')
if self.archive_compress:
filename = filename_orig + '.gz' + '.partial'
file = os.path.join(wals_dir, filename)
else:
filename = filename_orig + '.partial'
file = os.path.join(wals_dir, filename)
# emulate stale .partial file
with open(file, 'a') as f:
f.write(b"blahblah")
f.flush()
f.close()
self.switch_wal_segment(node)
sleep(20)
# check that segment is archived
if self.archive_compress:
filename_orig = filename_orig + '.gz'
file = os.path.join(wals_dir, filename_orig)
self.assertTrue(os.path.isfile(file))
# successful validate means that archive-push reused stale wal segment
self.validate_pb(
backup_dir, 'node',
options=['--recovery-target-xid={0}'.format(xid)])
# log_file = os.path.join(node.logs_dir, 'postgresql.log')
# with open(log_file, 'r') as f:
# log_content = f.read()
# self.assertIn(
# 'Reusing stale destination temporary WAL file',
# log_content)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_archive_push_partial_file_exists_not_stale(self):
"""Archive-push if .partial file exists and it is not stale"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
node.safe_psql(
"postgres",
"create table t1()")
self.switch_wal_segment(node)
node.safe_psql(
"postgres",
"create table t2()")
if self.get_version(node) < 100000:
filename_orig = node.safe_psql(
"postgres",
"SELECT file_name "
"FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip()
else:
filename_orig = node.safe_psql(
"postgres",
"SELECT file_name "
"FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip()
# form up path to next .partial WAL segment
wals_dir = os.path.join(backup_dir, 'wal', 'node')
if self.archive_compress:
filename = filename_orig + '.gz' + '.partial'
file = os.path.join(wals_dir, filename)
else:
filename = filename_orig + '.partial'
file = os.path.join(wals_dir, filename)
with open(file, 'a') as f:
f.write(b"blahblah")
f.flush()
f.close()
self.switch_wal_segment(node)
sleep(4)
with open(file, 'a') as f:
f.write(b"blahblahblahblah")
f.flush()
f.close()
sleep(10)
# check that segment is NOT archived
if self.archive_compress:
filename_orig = filename_orig + '.gz'
file = os.path.join(wals_dir, filename_orig)
self.assertFalse(os.path.isfile(file))
# log_file = os.path.join(node.logs_dir, 'postgresql.log')
# with open(log_file, 'r') as f:
# log_content = f.read()
# self.assertIn(
# 'is not stale',
# log_content)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_replica_archive(self):
tests/backup.py
+523 -5
View File
@@ -2,6 +2,7 @@ import unittest
import os
from time import sleep
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
import shutil
module_name = 'backup'
@@ -14,6 +15,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# PGPRO-707
def test_backup_modes_archive(self):
"""standart backup modes with ARCHIVE WAL method"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -108,6 +112,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_incremental_backup_without_full(self):
"""page-level backup without validated full backup"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -130,7 +137,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
"ERROR: Valid backup on current timeline is not found. "
"ERROR: Valid backup on current timeline 1 is not found. "
"Create new FULL backup before an incremental one.",
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
@@ -148,7 +155,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
"ERROR: Valid backup on current timeline is not found. "
"ERROR: Valid backup on current timeline 1 is not found. "
"Create new FULL backup before an incremental one.",
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
@@ -167,8 +174,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on'})
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
@@ -212,7 +218,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
"ERROR: Valid backup on current timeline is not found. "
"ERROR: Valid backup on current timeline 1 is not found. "
"Create new FULL backup before an incremental one.",
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
@@ -229,6 +235,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_ptrack_threads(self):
"""ptrack multi thread backup mode"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -257,6 +266,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_ptrack_threads_stream(self):
"""ptrack multi thread backup mode and stream"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -286,6 +298,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_page_corruption_heal_via_ptrack_1(self):
"""make node, corrupt some page, check that backup failed"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -342,6 +357,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_page_corruption_heal_via_ptrack_2(self):
"""make node, corrupt some page, check that backup failed"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -423,6 +441,137 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_backup_detect_corruption(self):
"""make node, corrupt some page, check that backup failed"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
node.safe_psql(
"postgres",
"create table t_heap as select 1 as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1000) i")
node.safe_psql(
"postgres",
"CHECKPOINT;")
heap_path = node.safe_psql(
"postgres",
"select pg_relation_filepath('t_heap')").rstrip()
node.stop()
with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f:
f.seek(9000)
f.write(b"bla")
f.flush()
f.close
node.slow_start()
try:
self.backup_node(
backup_dir, 'node', node,
backup_type="full", options=["-j", "4", "--stream"])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because tablespace mapping is incorrect"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
if self.ptrack:
self.assertTrue(
'WARNING: page verification failed, '
'calculated checksum' in e.message and
'ERROR: query failed: ERROR: '
'invalid page in block 1 of relation' in e.message and
'ERROR: Data files transferring failed' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
else:
if self.remote:
self.assertTrue(
"ERROR: Failed to read file" in e.message and
"data file checksum mismatch" in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
else:
self.assertIn(
'WARNING: Corruption detected in file',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
self.assertIn(
'ERROR: Data file corruption',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_backup_truncate_misaligned(self):
"""
make node, truncate file to size not even to BLCKSIZE,
take backup
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
node.safe_psql(
"postgres",
"create table t_heap as select 1 as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,100000) i")
node.safe_psql(
"postgres",
"CHECKPOINT;")
heap_path = node.safe_psql(
"postgres",
"select pg_relation_filepath('t_heap')").rstrip()
heap_size = node.safe_psql(
"postgres",
"select pg_relation_size('t_heap')")
with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f:
f.truncate(int(heap_size) - 4096)
f.flush()
f.close
output = self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"], return_id=False)
self.assertIn("WARNING: File", output)
self.assertIn("invalid file size", output)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_tablespace_in_pgdata_pgpro_1376(self):
"""PGPRO-1376 """
@@ -870,6 +1019,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_drop_rel_during_backup_ptrack(self):
""""""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
@@ -1365,3 +1517,369 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_backup_with_least_privileges_role(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'archive_timeout': '30s'})
if self.ptrack:
node.append_conf('postgresql.auto.conf', 'ptrack_enable = on')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
node.safe_psql(
'postgres',
'CREATE DATABASE backupdb')
# PG 9.5
if self.get_version(node) < 90600:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
# PG 9.6
elif self.get_version(node) > 90600 and self.get_version(node) < 100000:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
# >= 10
else:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
if self.ptrack:
for fname in [
'pg_catalog.oideq(oid, oid)',
'pg_catalog.ptrack_version()',
'pg_catalog.pg_ptrack_clear()',
'pg_catalog.pg_ptrack_control_lsn()',
'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)',
'pg_catalog.pg_ptrack_get_and_clear(oid, oid)',
'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)',
'pg_catalog.pg_stop_backup()']:
# try:
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION {0} "
"TO backup".format(fname))
# except:
# pass
# FULL backup
self.backup_node(
backup_dir, 'node', node,
datname='backupdb', options=['--stream', '-U', 'backup'])
self.backup_node(
backup_dir, 'node', node,
datname='backupdb', options=['-U', 'backup'])
# PAGE
self.backup_node(
backup_dir, 'node', node, backup_type='page',
datname='backupdb', options=['-U', 'backup'])
self.backup_node(
backup_dir, 'node', node, backup_type='page', datname='backupdb',
options=['--stream', '-U', 'backup'])
# DELTA
self.backup_node(
backup_dir, 'node', node, backup_type='delta',
datname='backupdb', options=['-U', 'backup'])
self.backup_node(
backup_dir, 'node', node, backup_type='delta',
datname='backupdb', options=['--stream', '-U', 'backup'])
# PTRACK
if self.ptrack:
self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
datname='backupdb', options=['-U', 'backup'])
self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
datname='backupdb', options=['--stream', '-U', 'backup'])
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_parent_choosing(self):
"""
PAGE3 <- RUNNING(parent should be FULL)
PAGE2 <- OK
PAGE1 <- CORRUPT
FULL
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
full_id = self.backup_node(backup_dir, 'node', node)
# PAGE1
page1_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# PAGE2
page2_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Change PAGE1 to ERROR
self.change_backup_status(backup_dir, 'node', page1_id, 'ERROR')
# PAGE3
page3_id = self.backup_node(
backup_dir, 'node', node,
backup_type='page', options=['--log-level-file=LOG'])
log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log')
with open(log_file_path) as f:
log_file_content = f.read()
self.assertIn(
"WARNING: Backup {0} has invalid parent: {1}. "
"Cannot be a parent".format(page2_id, page1_id),
log_file_content)
self.assertIn(
"WARNING: Backup {0} has status: ERROR. "
"Cannot be a parent".format(page1_id),
log_file_content)
self.assertIn(
"Parent backup: {0}".format(full_id),
log_file_content)
self.assertEqual(
self.show_pb(
backup_dir, 'node', backup_id=page3_id)['parent-backup-id'],
full_id)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_parent_choosing_1(self):
"""
PAGE3 <- RUNNING(parent should be FULL)
PAGE2 <- OK
PAGE1 <- (missing)
FULL
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
full_id = self.backup_node(backup_dir, 'node', node)
# PAGE1
page1_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# PAGE2
page2_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Delete PAGE1
shutil.rmtree(
os.path.join(backup_dir, 'backups', 'node', page1_id))
# PAGE3
page3_id = self.backup_node(
backup_dir, 'node', node,
backup_type='page', options=['--log-level-file=LOG'])
log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log')
with open(log_file_path) as f:
log_file_content = f.read()
self.assertIn(
"WARNING: Backup {0} has missing parent: {1}. "
"Cannot be a parent".format(page2_id, page1_id),
log_file_content)
self.assertIn(
"Parent backup: {0}".format(full_id),
log_file_content)
self.assertEqual(
self.show_pb(
backup_dir, 'node', backup_id=page3_id)['parent-backup-id'],
full_id)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_parent_choosing_2(self):
"""
PAGE3 <- RUNNING(backup should fail)
PAGE2 <- OK
PAGE1 <- OK
FULL <- (missing)
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
full_id = self.backup_node(backup_dir, 'node', node)
# PAGE1
page1_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# PAGE2
page2_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Delete FULL
shutil.rmtree(
os.path.join(backup_dir, 'backups', 'node', full_id))
# PAGE3
try:
self.backup_node(
backup_dir, 'node', node,
backup_type='page', options=['--log-level-file=LOG'])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because FULL backup is missing"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
'ERROR: Valid backup on current timeline 1 is not found. '
'Create new FULL backup before an incremental one.',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
self.assertEqual(
self.show_pb(
backup_dir, 'node')[2]['status'],
'ERROR')
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/checkdb.py
+4 -4
View File
@@ -229,7 +229,7 @@ class CheckdbTest(ProbackupTest, unittest.TestCase):
# create two databases
node.safe_psql("postgres", "create database db1")
try:
node.safe_psql(
node.safe_psql(
"db1",
"create extension amcheck")
except QueryException as e:
@@ -434,12 +434,12 @@ class CheckdbTest(ProbackupTest, unittest.TestCase):
repr(e.message), self.cmd))
self.assertIn(
"WARNING: CORRUPTION in file {0}, block 1".format(
'WARNING: Corruption detected in file "{0}", block 1'.format(
os.path.normpath(heap_full_path)),
e.message)
self.assertIn(
"WARNING: CORRUPTION in file {0}, block 5".format(
'WARNING: Corruption detected in file "{0}", block 5'.format(
os.path.normpath(heap_full_path)),
e.message)
@@ -484,7 +484,7 @@ class CheckdbTest(ProbackupTest, unittest.TestCase):
gdb.remove_all_breakpoints()
gdb._execute('signal SIGINT')
gdb.continue_execution_until_exit()
gdb.continue_execution_until_error()
with open(node.pg_log_file, 'r') as f:
output = f.read()
tests/compatibility.py
+11 -52
View File
@@ -217,6 +217,10 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_backward_compatibility_ptrack(self):
"""Description in jira issue PGPRO-434"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
@@ -224,8 +228,9 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'}
)
'autovacuum': 'off',
'ptrack_enable': 'on'})
self.init_pb(backup_dir, old_binary=True)
self.show_pb(backup_dir)
@@ -262,7 +267,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Delta BACKUP with old binary
# ptrack BACKUP with old binary
pgbench = node.pgbench(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
@@ -272,7 +277,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
pgbench.stdout.close()
self.backup_node(
backup_dir, 'node', node, backup_type='delta',
backup_dir, 'node', node, backup_type='ptrack',
old_binary=True)
if self.paranoia:
@@ -287,7 +292,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Delta BACKUP with new binary
# Ptrack BACKUP with new binary
pgbench = node.pgbench(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
@@ -297,7 +302,7 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
pgbench.stdout.close()
self.backup_node(
backup_dir, 'node', node, backup_type='delta')
backup_dir, 'node', node, backup_type='ptrack')
if self.paranoia:
pgdata = self.pgdata_content(node.data_dir)
@@ -532,49 +537,3 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_backup_concurrent_drop_table(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node, old_binary=True)
node.slow_start()
node.pgbench_init(scale=1)
# FULL backup
gdb = self.backup_node(
backup_dir, 'node', node,
options=['--stream', '--compress', '--log-level-file=VERBOSE'],
gdb=True, old_binary=True)
gdb.set_breakpoint('backup_data_file')
gdb.run_until_break()
node.safe_psql(
'postgres',
'DROP TABLE pgbench_accounts')
# do checkpoint to guarantee filenode removal
node.safe_psql(
'postgres',
'CHECKPOINT')
gdb.remove_all_breakpoints()
gdb.continue_execution_until_exit()
# show_backup = self.show_pb(backup_dir, 'node')[0]
# self.assertEqual(show_backup['status'], "OK")
# validate with fresh binary, it MUST be successful
self.validate_pb(backup_dir)
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/compression.py
+41 -61
View File
@@ -23,11 +23,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -59,15 +55,15 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
options=[
'--stream', '--compress-algorithm=zlib'])
# PTRACK BACKUP
# DELTA BACKUP
node.safe_psql(
"postgres",
"insert into t_heap select i as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(512,768) i")
ptrack_result = node.execute("postgres", "SELECT * FROM t_heap")
ptrack_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
delta_result = node.execute("postgres", "SELECT * FROM t_heap")
delta_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='delta',
options=['--stream', '--compress-algorithm=zlib'])
# Drop Node
@@ -105,11 +101,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(page_result, page_result_new)
node.cleanup()
# Check ptrack backup
# Check delta backup
self.assertIn(
"INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
"INFO: Restore of backup {0} completed.".format(delta_backup_id),
self.restore_node(
backup_dir, 'node', node, backup_id=ptrack_backup_id,
backup_dir, 'node', node, backup_id=delta_backup_id,
options=[
"-j", "4", "--immediate",
"--recovery-target-action=promote"]),
@@ -117,8 +113,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
node.slow_start()
ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(ptrack_result, ptrack_result_new)
delta_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(delta_result, delta_result_new)
node.cleanup()
# Clean after yourself
@@ -135,11 +131,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -167,14 +159,14 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, backup_type='page',
options=["--compress-algorithm=zlib"])
# PTRACK BACKUP
# DELTA BACKUP
node.safe_psql(
"postgres",
"insert into t_heap select i as id, md5(i::text) as text, "
"md5(i::text)::tsvector as tsvector from generate_series(0,3) i")
ptrack_result = node.execute("postgres", "SELECT * FROM t_heap")
ptrack_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
delta_result = node.execute("postgres", "SELECT * FROM t_heap")
delta_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='delta',
options=['--compress-algorithm=zlib'])
# Drop Node
@@ -212,11 +204,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(page_result, page_result_new)
node.cleanup()
# Check ptrack backup
# Check delta backup
self.assertIn(
"INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
"INFO: Restore of backup {0} completed.".format(delta_backup_id),
self.restore_node(
backup_dir, 'node', node, backup_id=ptrack_backup_id,
backup_dir, 'node', node, backup_id=delta_backup_id,
options=[
"-j", "4", "--immediate",
"--recovery-target-action=promote"]),
@@ -224,8 +216,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
node.slow_start()
ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(ptrack_result, ptrack_result_new)
delta_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(delta_result, delta_result_new)
node.cleanup()
# Clean after yourself
@@ -242,11 +234,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -275,15 +263,15 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, backup_type='page',
options=['--stream', '--compress-algorithm=pglz'])
# PTRACK BACKUP
# DELTA BACKUP
node.safe_psql(
"postgres",
"insert into t_heap select i as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(512,768) i")
ptrack_result = node.execute("postgres", "SELECT * FROM t_heap")
ptrack_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
delta_result = node.execute("postgres", "SELECT * FROM t_heap")
delta_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='delta',
options=['--stream', '--compress-algorithm=pglz'])
# Drop Node
@@ -321,11 +309,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(page_result, page_result_new)
node.cleanup()
# Check ptrack backup
# Check delta backup
self.assertIn(
"INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
"INFO: Restore of backup {0} completed.".format(delta_backup_id),
self.restore_node(
backup_dir, 'node', node, backup_id=ptrack_backup_id,
backup_dir, 'node', node, backup_id=delta_backup_id,
options=[
"-j", "4", "--immediate",
"--recovery-target-action=promote"]),
@@ -333,8 +321,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
node.slow_start()
ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(ptrack_result, ptrack_result_new)
delta_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(delta_result, delta_result_new)
node.cleanup()
# Clean after yourself
@@ -351,11 +339,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -384,15 +368,15 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, backup_type='page',
options=['--compress-algorithm=pglz'])
# PTRACK BACKUP
# DELTA BACKUP
node.safe_psql(
"postgres",
"insert into t_heap select i as id, md5(i::text) as text, "
"md5(i::text)::tsvector as tsvector "
"from generate_series(200,300) i")
ptrack_result = node.execute("postgres", "SELECT * FROM t_heap")
ptrack_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
delta_result = node.execute("postgres", "SELECT * FROM t_heap")
delta_backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='delta',
options=['--compress-algorithm=pglz'])
# Drop Node
@@ -430,11 +414,11 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
self.assertEqual(page_result, page_result_new)
node.cleanup()
# Check ptrack backup
# Check delta backup
self.assertIn(
"INFO: Restore of backup {0} completed.".format(ptrack_backup_id),
"INFO: Restore of backup {0} completed.".format(delta_backup_id),
self.restore_node(
backup_dir, 'node', node, backup_id=ptrack_backup_id,
backup_dir, 'node', node, backup_id=delta_backup_id,
options=[
"-j", "4", "--immediate",
"--recovery-target-action=promote"]),
@@ -442,8 +426,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
repr(self.output), self.cmd))
node.slow_start()
ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(ptrack_result, ptrack_result_new)
delta_result_new = node.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(delta_result, delta_result_new)
node.cleanup()
# Clean after yourself
@@ -460,11 +444,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s',
'ptrack_enable': 'on'}
)
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
tests/delete.py
+103 -28
View File
@@ -89,7 +89,61 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_delete_archive_mix_compress_and_non_compressed_segments(self):
"""stub"""
"""delete full backups"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir="{0}/{1}/node".format(module_name, fname),
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(
backup_dir, 'node', node, compress=False)
node.slow_start()
# full backup
self.backup_node(backup_dir, 'node', node)
node.pgbench_init(scale=10)
# Restart archiving with compression
self.set_archiving(backup_dir, 'node', node, compress=True)
node.restart()
# full backup
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
self.backup_node(
backup_dir, 'node', node,
options=[
'--retention-redundancy=3',
'--delete-expired'])
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
self.backup_node(
backup_dir, 'node', node,
options=[
'--retention-redundancy=3',
'--delete-expired'])
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
self.backup_node(
backup_dir, 'node', node,
options=[
'--retention-redundancy=3',
'--delete-expired'])
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_delete_increment_page(self):
@@ -134,6 +188,9 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_delete_increment_ptrack(self):
"""delete increment and all after him"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -358,11 +415,12 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
backup_id_a = self.backup_node(backup_dir, 'node', node)
backup_id_b = self.backup_node(backup_dir, 'node', node)
# Change FULL B backup status to ERROR
# Change FULLb to ERROR
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# FULLb ERROR
# FULLa OK
# Take PAGEa1 backup
page_id_a1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -370,15 +428,17 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# PAGEa1 OK
# FULLb ERROR
# FULLa OK
# Change FULL B backup status to OK
# Change FULLb to OK
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa1 backup status to ERROR
# Change PAGEa1 to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR')
# PAGEa1 ERROR
# FULLb OK
# FULLa OK
page_id_b1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -386,41 +446,49 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# PAGEa1 ERROR
# FULLb OK
# FULLa OK
# Now we start to play with first generation of PAGE backups
# Change PAGEb1 status to ERROR
# Change PAGEb1 and FULLb status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# Change PAGEa1 status to OK
self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK')
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# PAGEa2 OK
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEa2 status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
# Change PAGEb1 status to OK
# Change PAGEa2 and FULla to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# Change PAGEb1 and FULlb to OK
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# PAGEa2 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Change PAGEa2 status to OK
# Change PAGEa2 and FULLa status to OK
self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# PAGEb2 OK
# PAGEa2 OK
@@ -478,16 +546,15 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# Take FULL BACKUPs
backup_id_a = self.backup_node(backup_dir, 'node', node)
backup_id_b = self.backup_node(backup_dir, 'node', node)
# Change FULLb backup status to ERROR
# Change FULLb to ERROR
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
page_id_a1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Change FULLb backup status to OK
# Change FULLb to OK
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa1 backup status to ERROR
@@ -505,15 +572,16 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# FULLb OK
# FULLa OK
# Change PAGEa1 backup status to OK
# Change PAGEa1 to OK
self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK')
# Change PAGEb1 backup status to ERROR
# Change PAGEb1 and FULLb backup status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a2 = self.backup_node(
@@ -522,20 +590,22 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# PAGEa2 OK
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEb1 backup status to OK
# Change PAGEb1 and FULLb to OK
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa2 backup status to ERROR
# Change PAGEa2 and FULLa to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# PAGEa2 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -545,17 +615,21 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
# Change PAGEb2 and PAGEb1 status to ERROR
# Change PAGEb2, PAGEb1 and FULLb to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR')
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# Change FULLa to OK
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
# PAGEb2 ERROR
# PAGEa2 ERROR
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a3 = self.backup_node(
@@ -566,14 +640,15 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# PAGEa2 ERROR
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEa3 status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR')
# Change PAGEb2 status to OK
# Change PAGEb2 and FULLb to OK
self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
page_id_b3 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -587,7 +662,7 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# FULLb OK
# FULLa OK
# Change PAGEa3, PAGEa2 and PAGEb1 status to OK
# Change PAGEa3, PAGEa2 and PAGEb1 to OK
self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK')
self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK')
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
tests/delta.py
+6
View File
@@ -1078,6 +1078,9 @@ class DeltaTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_delta_corruption_heal_via_ptrack_1(self):
"""make node, corrupt some page, check that backup failed"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -1135,6 +1138,9 @@ class DeltaTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_page_corruption_heal_via_ptrack_2(self):
"""make node, corrupt some page, check that backup failed"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
tests/exclude.py
+4 -8
View File
@@ -20,10 +20,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'max_wal_senders': '2',
'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'})
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -102,7 +99,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase):
def test_exclude_unlogged_tables_1(self):
"""
make node without archiving, create unlogged table, take full backup,
alter table to unlogged, take ptrack backup, restore ptrack backup,
alter table to unlogged, take delta backup, restore delta backup,
check that PGDATA`s are physically the same
"""
fname = self.id().split('.')[3]
@@ -113,8 +110,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase):
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off',
"shared_buffers": "10MB",
'ptrack_enable': 'on'})
"shared_buffers": "10MB"})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@@ -138,7 +134,7 @@ class ExcludeTest(ProbackupTest, unittest.TestCase):
node.safe_psql('postgres', "alter table test set logged")
self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
backup_dir, 'node', node, backup_type='delta',
options=['--stream']
)
tests/expected/option_help.out
+13
View File
@@ -26,9 +26,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]
[--help]
pg_probackup show-config -B backup-path --instance=instance_name
[--format=format]
[--help]
pg_probackup backup -B backup-path -b backup-mode --instance=instance_name
[-D pgdata-path] [-C]
@@ -55,6 +57,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]
[--help]
pg_probackup restore -B backup-path --instance=instance_name
[-D pgdata-path] [-i backup-id] [-j num-threads]
@@ -72,6 +75,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]
[--help]
pg_probackup validate -B backup-path [--instance=instance_name]
[-i backup-id] [--progress] [-j num-threads]
@@ -80,22 +84,27 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[--recovery-target-timeline=timeline]
[--recovery-target-name=target-name]
[--skip-block-validation]
[--help]
pg_probackup checkdb [-B backup-path] [--instance=instance_name]
[-D pgdata-path] [--progress] [-j num-threads]
[--amcheck] [--skip-block-validation]
[--heapallindexed]
[--help]
pg_probackup show -B backup-path
[--instance=instance_name [-i backup-id]]
[--format=format]
[--help]
pg_probackup delete -B backup-path --instance=instance_name
[--wal] [-i backup-id | --expired | --merge-expired]
[--dry-run]
[--help]
pg_probackup merge -B backup-path --instance=instance_name
-i backup-id [--progress] [-j num-threads]
[--help]
pg_probackup add-instance -B backup-path -D pgdata-path
--instance=instance_name
@@ -103,9 +112,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]
[--help]
pg_probackup del-instance -B backup-path
--instance=instance_name
[--help]
pg_probackup archive-push -B backup-path --instance=instance_name
--wal-file-path=wal-file-path
@@ -117,6 +128,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]
[--help]
pg_probackup archive-get -B backup-path --instance=instance_name
--wal-file-path=wal-file-path
@@ -124,6 +136,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]
[--help]
Read the website for details. <https://github.com/postgrespro/pg_probackup>
Report bugs to <https://github.com/postgrespro/pg_probackup/issues>.
tests/expected/option_version.out
+1 -1
View File
@@ -1 +1 @@
pg_probackup 2.1.3
pg_probackup 2.1.4
tests/false_positive.py
+9 -2
View File
@@ -57,8 +57,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on'})
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
@@ -114,6 +113,10 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
def test_ptrack_concurrent_get_and_clear_1(self):
"""make node, make full and ptrack stream backups,"
" restore them and check data correctness"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
@@ -195,6 +198,10 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
def test_ptrack_concurrent_get_and_clear_2(self):
"""make node, make full and ptrack stream backups,"
" restore them and check data correctness"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
tests/helpers/ptrack_helpers.py
+9 -1
View File
@@ -271,7 +271,15 @@ class ProbackupTest(object):
self.remote_user = None
if 'PGPROBACKUP_SSH_REMOTE' in self.test_env:
self.remote = True
if self.test_env['PGPROBACKUP_SSH_REMOTE'] == 'ON':
self.remote = True
self.ptrack = False
if 'PG_PROBACKUP_PTRACK' in self.test_env:
if self.test_env['PG_PROBACKUP_PTRACK'] == 'ON':
self.ptrack = True
os.environ["PGAPPNAME"] = "pg_probackup"
@property
def pg_config_version(self):
tests/logging.py
+3 -4
View File
@@ -142,7 +142,7 @@ class LogTest(ProbackupTest, unittest.TestCase):
log_file_size)
self.assertNotIn(
'WARNING:',
'WARNING: cannot read creation timestamp from rotation file',
output)
self.assertTrue(os.path.isfile(rotation_file_path))
@@ -166,7 +166,6 @@ class LogTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node',
options=['--log-rotation-age=1d'])
self.backup_node(
backup_dir, 'node', node,
options=[
@@ -212,7 +211,7 @@ class LogTest(ProbackupTest, unittest.TestCase):
return_id=False)
self.assertNotIn(
'WARNING:',
'WARNING: missing rotation file:',
output)
# check that log file wasn`t rotated
@@ -291,7 +290,7 @@ class LogTest(ProbackupTest, unittest.TestCase):
return_id=False)
self.assertNotIn(
'WARNING:',
'WARNING: rotation file',
output)
# check that log file wasn`t rotated
tests/merge.py
+48 -25
View File
@@ -826,6 +826,9 @@ class MergeTest(ProbackupTest, unittest.TestCase):
take page backup, merge full and page,
restore last page backup and check data correctness
"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
@@ -1651,8 +1654,6 @@ class MergeTest(ProbackupTest, unittest.TestCase):
os.remove(file_to_remove)
# Try to continue failed MERGE
#print(backup_id)
#exit(1)
self.merge_backup(backup_dir, "node", backup_id)
self.assertEqual(
@@ -1875,15 +1876,28 @@ class MergeTest(ProbackupTest, unittest.TestCase):
pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum'])
pgbench.wait()
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgdata = self.pgdata_content(node.data_dir)
node.cleanup()
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
self.restore_node(
backup_dir, 'node',
node_restored, backup_id=backup_id)
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# check that merged backup has the same state as
node_restored.cleanup()
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
self.restore_node(backup_dir, 'node', node, backup_id=backup_id)
pgdata_restored = self.pgdata_content(node.data_dir)
self.restore_node(
backup_dir, 'node',
node_restored, backup_id=backup_id)
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
@@ -1941,15 +1955,16 @@ class MergeTest(ProbackupTest, unittest.TestCase):
# FULLb OK
# FULLa OK
# Change PAGEa1 backup status to OK
# Change PAGEa1 to OK
self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK')
# Change PAGEb1 backup status to ERROR
# Change PAGEb1 and FULLb to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a2 = self.backup_node(
@@ -1958,20 +1973,22 @@ class MergeTest(ProbackupTest, unittest.TestCase):
# PAGEa2 OK
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEb1 backup status to OK
# Change PAGEb1 and FULLb to OK
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa2 backup status to ERROR
# Change PAGEa2 and FULL to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# PAGEa2 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -1981,17 +1998,21 @@ class MergeTest(ProbackupTest, unittest.TestCase):
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
# Change PAGEb2 and PAGEb1 status to ERROR
# Change PAGEb2, PAGEb1 and FULLb to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR')
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# Change FULLa to OK
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
# PAGEb2 ERROR
# PAGEa2 ERROR
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a3 = self.backup_node(
@@ -2002,14 +2023,16 @@ class MergeTest(ProbackupTest, unittest.TestCase):
# PAGEa2 ERROR
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEa3 status to ERROR
# Change PAGEa3 and FULLa to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR')
# Change PAGEb2 status to OK
# Change PAGEb2, PAGEb1 and FULLb to OK
self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK')
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
page_id_b3 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -2018,15 +2041,15 @@ class MergeTest(ProbackupTest, unittest.TestCase):
# PAGEa3 ERROR
# PAGEb2 OK
# PAGEa2 ERROR
# PAGEb1 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
# Change PAGEa3, PAGEa2 and PAGEb1 status to OK
# Change PAGEa3, PAGEa2 and FULLa status to OK
self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK')
self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK')
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
# PAGEb3 OK
# PAGEa3 OK
tests/pgpro560.py
+49 -20
View File
@@ -1,6 +1,6 @@
import os
import unittest
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from datetime import datetime, timedelta
import subprocess
@@ -30,19 +30,22 @@ class CheckSystemID(ProbackupTest, unittest.TestCase):
self.add_instance(backup_dir, 'node', node)
node.slow_start()
file = os.path.join(node.base_dir,'data', 'global', 'pg_control')
file = os.path.join(node.base_dir, 'data', 'global', 'pg_control')
os.remove(file)
try:
self.backup_node(backup_dir, 'node', node, options=['--stream'])
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
self.assertEqual(
1, 0,
"Expecting Error because pg_control was deleted.\n "
"Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: could not open file' in e.message
and 'pg_control' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
'ERROR: could not open file' in e.message and
'pg_control' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
@@ -75,24 +78,50 @@ class CheckSystemID(ProbackupTest, unittest.TestCase):
try:
self.backup_node(backup_dir, 'node1', node2, options=['--stream'])
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
self.assertEqual(
1, 0,
"Expecting Error because of SYSTEM ID mismatch.\n "
"Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: Backup data directory was initialized for system id' in e.message
and 'but connected instance system id is' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
if self.get_version(node1) > 90600:
self.assertTrue(
'ERROR: Backup data directory was '
'initialized for system id' in e.message and
'but connected instance system id is' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
else:
self.assertIn(
'ERROR: System identifier mismatch. '
'Connected PostgreSQL instance has system id',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
try:
self.backup_node(backup_dir, 'node1', node2, data_dir=node1.data_dir, options=['--stream'])
self.backup_node(
backup_dir, 'node1', node2,
data_dir=node1.data_dir, options=['--stream'])
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
self.assertEqual(
1, 0,
"Expecting Error because of of SYSTEM ID mismatch.\n "
"Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: Backup data directory was initialized for system id' in e.message
and 'but connected instance system id is' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
if self.get_version(node1) > 90600:
self.assertTrue(
'ERROR: Backup data directory was initialized '
'for system id' in e.message and
'but connected instance system id is' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
else:
self.assertIn(
'ERROR: System identifier mismatch. '
'Connected PostgreSQL instance has system id',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/replica.py
+91
View File
@@ -18,6 +18,9 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
make node, take full backup, restore it and make replica from it,
take full stream backup from replica
"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
master = self.make_simple_node(
@@ -437,3 +440,91 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_replica_promote(self):
"""
start backup from replica, during backup promote replica
check that backup is failed
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
master = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'master'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'archive_timeout': '10s',
'checkpoint_timeout': '30s',
'max_wal_size': '32MB'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'master', master)
self.set_archiving(backup_dir, 'master', master)
master.slow_start()
replica = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'replica'))
replica.cleanup()
self.backup_node(backup_dir, 'master', master)
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,165000) i")
self.restore_node(
backup_dir, 'master', replica, options=['-R'])
# Settings for Replica
self.add_instance(backup_dir, 'replica', replica)
self.set_archiving(backup_dir, 'replica', replica, replica=True)
self.set_replica(
master, replica,
replica_name='replica', synchronous=True)
replica.slow_start(replica=True)
master.psql(
"postgres",
"create table t_heap_1 as select i as id, md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,165000) i")
self.wait_until_replica_catch_with_master(master, replica)
# start backup from replica
gdb = self.backup_node(
backup_dir, 'replica', replica, gdb=True,
options=['--log-level-file=verbose'])
gdb.set_breakpoint('backup_data_file')
gdb.run_until_break()
gdb.continue_execution_until_break(20)
replica.promote()
gdb.remove_all_breakpoints()
gdb.continue_execution_until_exit()
backup_id = self.show_pb(
backup_dir, 'replica')[0]["id"]
# read log file content
with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f:
log_content = f.read()
f.close
self.assertIn(
'ERROR: the standby was promoted during online backup',
log_content)
self.assertIn(
'WARNING: Backup {0} is running, '
'setting its status to ERROR'.format(backup_id),
log_content)
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/restore.py
+44 -27
View File
@@ -492,6 +492,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_restore_full_ptrack_archive(self):
"""recovery to latest from archive full+ptrack backups"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -540,6 +543,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_restore_ptrack(self):
"""recovery to latest from archive full+ptrack+ptrack backups"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -595,6 +601,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_restore_full_ptrack_stream(self):
"""recovery in stream mode to latest from full + ptrack backups"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -647,6 +656,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
recovery to latest from full + ptrack backups
with loads when ptrack backup do
"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -711,6 +723,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
recovery to latest from full + page backups
with loads when full backup do
"""
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
@@ -2028,33 +2043,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.slow_start()
# Restore with recovery target lsn
node.cleanup()
self.restore_node(
backup_dir, 'node', node,
options=[
'--recovery-target-lsn={0}'.format(target_lsn),
"--recovery-target-action=promote",
'--recovery-target-timeline=1',
])
with open(recovery_conf, 'r') as f:
recovery_conf_content = f.read()
self.assertIn(
"recovery_target_lsn = '{0}'".format(target_lsn),
recovery_conf_content)
self.assertIn(
"recovery_target_action = 'promote'",
recovery_conf_content)
self.assertIn(
"recovery_target_timeline = '1'",
recovery_conf_content)
node.slow_start()
# Restore with recovery target name
node.cleanup()
self.restore_node(
@@ -2082,6 +2070,35 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.slow_start()
# Restore with recovery target lsn
if self.get_version(node) >= 100000:
node.cleanup()
self.restore_node(
backup_dir, 'node', node,
options=[
'--recovery-target-lsn={0}'.format(target_lsn),
"--recovery-target-action=promote",
'--recovery-target-timeline=1',
])
with open(recovery_conf, 'r') as f:
recovery_conf_content = f.read()
self.assertIn(
"recovery_target_lsn = '{0}'".format(target_lsn),
recovery_conf_content)
self.assertIn(
"recovery_target_action = 'promote'",
recovery_conf_content)
self.assertIn(
"recovery_target_timeline = '1'",
recovery_conf_content)
node.slow_start()
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/retention.py
+178 -93
View File
@@ -2,6 +2,7 @@ import os
import unittest
from datetime import datetime, timedelta
from .helpers.ptrack_helpers import ProbackupTest
from time import sleep
module_name = 'retention'
@@ -246,11 +247,12 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
backup_id_a = self.backup_node(backup_dir, 'node', node)
backup_id_b = self.backup_node(backup_dir, 'node', node)
# Change FULL B backup status to ERROR
# Change FULLb backup status to ERROR
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# FULLb ERROR
# FULLa OK
# Take PAGEa1 backup
page_id_a1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -258,57 +260,69 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# PAGEa1 OK
# FULLb ERROR
# FULLa OK
# Change FULL B backup status to OK
# Change FULLb backup status to OK
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa1 backup status to ERROR
# Change PAGEa1 and FULLa to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# PAGEa1 ERROR
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# PAGEb1 OK
# PAGEa1 ERROR
# FULLb OK
# FULLa OK
# Now we start to play with first generation of PAGE backups
# Change PAGEb1 status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
# FULLa ERROR
# Change PAGEa1 status to OK
# Now we start to play with first generation of PAGE backups
# Change PAGEb1 and FULLb to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# Change PAGEa1 and FULLa to OK
self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# PAGEa2 OK
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEa2 status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
# Change PAGEb1 status to OK
# Change PAGEa2 and FULLa to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# Change PAGEb1 and FULLb to OK
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# PAGEa2 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Change PAGEa2 status to OK
# Change PAGEa2 and FULla to OK
self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
# PAGEb2 OK
# PAGEa2 OK
@@ -320,14 +334,12 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# Purge backups
backups = os.path.join(backup_dir, 'backups', 'node')
for backup in os.listdir(backups):
if backup in [page_id_a2, page_id_b2, 'pg_probackup.conf']:
continue
with open(
os.path.join(
backups, backup, "backup.control"), "a") as conf:
conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format(
datetime.now() - timedelta(days=3)))
if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']:
with open(
os.path.join(
backups, backup, "backup.control"), "a") as conf:
conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format(
datetime.now() - timedelta(days=3)))
self.delete_expired(
backup_dir, 'node',
@@ -371,32 +383,38 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# PAGEa1 OK
# FULLb ERROR
# FULLa OK
# Change FULL B backup status to OK
# Change FULLb backup status to OK
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa1 backup status to ERROR
# Change PAGEa1 and FULLa backup status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# PAGEa1 ERROR
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# PAGEb1 OK
# PAGEa1 ERROR
# FULLb OK
# FULLa OK
# FULLa ERROR
# Now we start to play with first generation of PAGE backups
# Change PAGEb1 status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# Change PAGEa1 status to OK
self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -404,24 +422,28 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# PAGEa2 OK
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEa2 status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
# Change PAGEb1 status to OK
# Change PAGEa2 and FULLa status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# Change PAGEb1 and FULLb status to OK
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# PAGEa2 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Change PAGEa2 status to OK
# Change PAGEa2 and FULLa status to OK
self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
# PAGEb2 OK
# PAGEa2 OK
@@ -460,11 +482,12 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
backup_id_a = self.backup_node(backup_dir, 'node', node)
backup_id_b = self.backup_node(backup_dir, 'node', node)
# Change FULL B backup status to ERROR
# Change FULLb backup status to ERROR
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# FULLb ERROR
# FULLa OK
# Take PAGEa1 backup
page_id_a1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -472,7 +495,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# PAGEa1 OK
# FULLb ERROR
# FULLa OK
# Change FULL B backup status to OK
# Change FULLb to OK
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa1 backup status to ERROR
@@ -481,6 +505,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# PAGEa1 ERROR
# FULLb OK
# FULLa OK
page_id_b1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -488,41 +513,49 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# PAGEa1 ERROR
# FULLb OK
# FULLa OK
# Now we start to play with first generation of PAGE backups
# Change PAGEb1 status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
# Change PAGEa1 status to OK
# Now we start to play with first generation of PAGE backups
# Change PAGEb1 and FULLb to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# Change PAGEa1 to OK
self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK')
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# PAGEa2 OK
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEa2 status to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
# Change PAGEb1 status to OK
# Change PAGEa2 and FULLa to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# Change PAGEb1 and FULLb to OK
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# PAGEa2 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# Change PAGEa2 status to OK
# Change PAGEa2 and FULLa to OK
self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
# PAGEb2 OK
# PAGEa2 OK
@@ -534,14 +567,12 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# Purge backups
backups = os.path.join(backup_dir, 'backups', 'node')
for backup in os.listdir(backups):
if backup in [page_id_a2, page_id_b2, 'pg_probackup.conf']:
continue
with open(
os.path.join(
backups, backup, "backup.control"), "a") as conf:
conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format(
datetime.now() - timedelta(days=3)))
if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']:
with open(
os.path.join(
backups, backup, "backup.control"), "a") as conf:
conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format(
datetime.now() - timedelta(days=3)))
output = self.delete_expired(
backup_dir, 'node',
@@ -767,12 +798,12 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# Take FULL BACKUPs
backup_id_a = self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
# pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
# pgbench.wait()
backup_id_b = self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
# pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
# pgbench.wait()
# Change FULLb backup status to ERROR
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
@@ -780,13 +811,13 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
page_id_a1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
# pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
# pgbench.wait()
# Change FULLb backup status to OK
# Change FULLb to OK
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa1 backup status to ERROR
# Change PAGEa1 to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR')
# PAGEa1 ERROR
@@ -801,86 +832,95 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# FULLb OK
# FULLa OK
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
# pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
# pgbench.wait()
# Change PAGEa1 backup status to OK
# Change PAGEa1 to OK
self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK')
# Change PAGEb1 backup status to ERROR
# Change PAGEb1 and FULLb to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
# pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
# pgbench.wait()
# PAGEa2 OK
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEb1 backup status to OK
# Change PAGEb1 and FULLb to OK
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
# Change PAGEa2 backup status to ERROR
# Change PAGEa2 and FULLa to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR')
# PAGEa2 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
page_id_b2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
# pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
# pgbench.wait()
# PAGEb2 OK
# PAGEa2 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
# FULLa ERROR
# Change PAGEb2 and PAGEb1 status to ERROR
# Change PAGEb2 and PAGEb1 to ERROR
self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR')
self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR')
# and FULL stuff
self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
# PAGEb2 ERROR
# PAGEa2 ERROR
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
page_id_a3 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
pgbench.wait()
# pgbench = node.pgbench(options=['-T', '10', '-c', '2'])
# pgbench.wait()
# PAGEa3 OK
# PAGEb2 ERROR
# PAGEa2 ERROR
# PAGEb1 ERROR
# PAGEa1 OK
# FULLb OK
# FULLb ERROR
# FULLa OK
# Change PAGEa3 status to ERROR
# Change PAGEa3 to ERROR
self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR')
# Change PAGEb2 status to OK
# Change PAGEb2, PAGEb1 and FULLb to OK
self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK')
self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK')
page_id_b3 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
@@ -889,7 +929,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# PAGEa3 ERROR
# PAGEb2 OK
# PAGEa2 ERROR
# PAGEb1 ERROR
# PAGEb1 OK
# PAGEa1 OK
# FULLb OK
# FULLa OK
@@ -910,16 +950,15 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1
self.assertEqual(
self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'],
self.show_pb(
backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'],
page_id_a1)
self.assertEqual(
self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'],
self.show_pb(
backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'],
page_id_a1)
print("Backups {0} and {1} are children of {2}".format(
page_id_a3, page_id_a2, page_id_a1))
# Purge backups
backups = os.path.join(backup_dir, 'backups', 'node')
for backup in os.listdir(backups):
@@ -1202,7 +1241,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
@unittest.skip("skip")
def test_window_error_backups(self):
"""
PAGE ERROR
@@ -1233,4 +1272,50 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, backup_type='page')
# Change FULLb backup status to ERROR
# self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
def test_retention_redundancy_overlapping_chains(self):
""""""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'])
if self.get_version(node) < 90600:
self.del_test_dir(module_name, fname)
return unittest.skip('Skipped because ptrack support is disabled')
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
self.set_config(
backup_dir, 'node', options=['--retention-redundancy=1'])
# Make backups to be purged
self.backup_node(backup_dir, 'node', node)
self.backup_node(backup_dir, 'node', node, backup_type="page")
# Make backups to be keeped
gdb = self.backup_node(backup_dir, 'node', node, gdb=True)
gdb.set_breakpoint('backup_files')
gdb.run_until_break()
sleep(1)
self.backup_node(backup_dir, 'node', node, backup_type="page")
gdb.remove_all_breakpoints()
gdb.continue_execution_until_exit()
self.backup_node(backup_dir, 'node', node, backup_type="page")
# Purge backups
log = self.delete_expired(
backup_dir, 'node', options=['--expired', '--wal'])
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2)
# Clean after yourself
self.del_test_dir(module_name, fname)