db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Merge branch 'REL_2_5' before PBCKP-146 into try-merge-2.6

Browse Source
This commit is contained in:
Yura Sokolov
2022-11-07 19:48:47 +03:00
parent e05a8c3634 d25a594df5
commit 46c782b012
32 changed files with 1017 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+3 -1
View File
@@ -26,7 +26,8 @@ notifications:
# Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON
env:
- PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master
- PG_VERSION=16 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master
- PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=REL_15_STABLE
- PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE
- PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE
- PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE
@@ -35,6 +36,7 @@ env:
# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive
# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup
# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup
# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb
# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=compression
# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta
# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=locking
README.md
+18 -18
View File
@@ -6,7 +6,7 @@
`pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure.
The utility is compatible with:
* PostgreSQL 10, 11, 12, 13, 14;
* PostgreSQL 10, 11, 12, 13, 14, 15;
As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data:
* Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow.
@@ -41,9 +41,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
## ptrack support
`PTRACK` backup support provided via following options:
* vanilla PostgreSQL 11, 12, 13, 14 with [ptrack extension](https://github.com/postgrespro/ptrack)
* Postgres Pro Standard 11, 12, 13
* Postgres Pro Enterprise 11, 12, 13
* vanilla PostgreSQL 11, 12, 13, 14, 15 with [ptrack extension](https://github.com/postgrespro/ptrack)
* Postgres Pro Standard 11, 12, 13, 14
* Postgres Pro Enterprise 11, 12, 13, 14
## Limitations
@@ -137,8 +137,8 @@ sudo apt-get install pg_probackup-{14,13,12,11,10}-debuginfo
#DEB Ubuntu|Debian Packages
sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list'
sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update
sudo apt-get install pg-probackup-{std,ent}-{13,12,11,10}
sudo apt-get install pg-probackup-{std,ent}-{13,12,11,10}-dbg
sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10}
sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10}-dbg
#DEB Astra Linix Orel
sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list'
@@ -148,35 +148,35 @@ sudo apt-get install pg-probackup-{std,ent}-{12,11,10}{-dbg,}
#RPM Centos Packages
rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm
yum install pg_probackup-{std,ent}-{13,12,11,10}
yum install pg_probackup-{std,ent}-{13,12,11,10}-debuginfo
yum install pg_probackup-{std,ent}-{14,13,12,11,10}
yum install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo
#RPM RHEL Packages
rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm
yum install pg_probackup-{std,ent}-{13,12,11,10}
yum install pg_probackup-{std,ent}-{13,12,11,10}-debuginfo
yum install pg_probackup-{std,ent}-{14,13,12,11,10}
yum install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo
#RPM Oracle Linux Packages
rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm
yum install pg_probackup-{std,ent}-{13,12,11,10}
yum install pg_probackup-{std,ent}-{13,12,11,10}-debuginfo
yum install pg_probackup-{std,ent}-{14,13,12,11,10}
yum install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo
#RPM ALT Linux 7
sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list'
sudo apt-get update
sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10}
sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10}-debuginfo
sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}
sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo
#RPM ALT Linux 8
sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list'
sudo apt-get update
sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10}
sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10}-debuginfo
sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}
sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo
#RPM ALT Linux 9
sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update
sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10}
sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10}-debuginfo
sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}
sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo
```
Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup).
doc/pgprobackup.xml
+22 -1
View File
@@ -618,7 +618,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup;
COMMIT;
</programlisting>
<para>
For <productname>PostgreSQL</productname> 10 or higher:
For <productname>PostgreSQL</productname> 10:
</para>
<programlisting>
BEGIN;
@@ -637,6 +637,27 @@ GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup;
COMMIT;
</programlisting>
<para>
For <productname>PostgreSQL</productname> 15 or higher:
</para>
<programlisting>
BEGIN;
CREATE ROLE backup WITH LOGIN;
GRANT USAGE ON SCHEMA pg_catalog TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;
GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup;
COMMIT;
</programlisting>
<para>
In the
packaging/Makefile.test
+2 -2
View File
@@ -130,10 +130,10 @@ build/test_suse: build/test_suse_15.1 build/test_suse_15.2
@echo Suse: done
build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13
@echo Rhel 15.1: done
@echo Suse 15.1: done
build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 build/test_suse_15.2_14
@echo Rhel 15.1: done
@echo Suse 15.2: done
define test_suse
docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \
packaging/test/scripts/rpm.sh
+6
View File
@@ -77,6 +77,12 @@ if [ ${DISTRIB} == 'centos' ] && [ ${DISTRIB_VERSION} == '8' ]; then
dnf -qy module disable postgresql
fi
# PGDG doesn't support install of PG-9.6 from repo package anymore
if [ ${PG_VERSION} == '9.6' ] && [ ${DISTRIB_VERSION} == '7' ]; then
# ugly hack: use repo settings from PG10
sed -i 's/10/9.6/' /etc/yum.repos.d/pgdg-redhat-all.repo
fi
yum install -y postgresql${PG_TOG}-server.x86_64
export PGDATA=/var/lib/pgsql/${PG_VERSION}/data
src/backup.c
+60 -3
View File
@@ -923,10 +923,21 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo)
*/
#ifdef PGPRO_VERSION
if (!res)
{
/* It seems we connected to PostgreSQL (not Postgres Pro) */
if(strcmp(PGPRO_EDITION, "1C") != 0)
{
elog(ERROR, "%s was built with Postgres Pro %s %s, "
"but connection is made with PostgreSQL %s",
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
}
/* We have PostgresPro for 1C and connect to PostgreSQL or PostgresPro for 1C
* Check the major version
*/
if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0)
elog(ERROR, "%s was built with PostgrePro %s %s, but connection is made with %s",
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
}
else
{
if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 &&
@@ -1030,12 +1041,20 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
uint32 lsn_lo;
params[0] = label;
#if PG_VERSION_NUM >= 150000
elog(INFO, "wait for pg_backup_start()");
#else
elog(INFO, "wait for pg_start_backup()");
#endif
/* 2nd argument is 'fast'*/
params[1] = smooth ? "false" : "true";
res = pgut_execute(conn,
#if PG_VERSION_NUM >= 150000
"SELECT pg_catalog.pg_backup_start($1, $2)",
#else
"SELECT pg_catalog.pg_start_backup($1, $2, false)",
#endif
2,
params);
@@ -1575,6 +1594,14 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica
" labelfile,"
" spcmapfile"
" FROM pg_catalog.pg_stop_backup(false, false)",
stop_backup_on_master_after15_query[] =
"SELECT"
" pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot()),"
" current_timestamp(0)::timestamptz,"
" lsn,"
" labelfile,"
" spcmapfile"
" FROM pg_catalog.pg_backup_stop(false)",
/*
* In case of backup from replica we do not trust minRecPoint
* and stop_backup LSN, so we use latest replayed LSN as STOP LSN.
@@ -1586,12 +1613,26 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica
" pg_catalog.pg_last_wal_replay_lsn(),"
" labelfile,"
" spcmapfile"
" FROM pg_catalog.pg_stop_backup(false, false)";
" FROM pg_catalog.pg_stop_backup(false, false)",
stop_backup_on_replica_after15_query[] =
"SELECT"
" pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot()),"
" current_timestamp(0)::timestamptz,"
" pg_catalog.pg_last_wal_replay_lsn(),"
" labelfile,"
" spcmapfile"
" FROM pg_catalog.pg_backup_stop(false)";
const char * const stop_backup_query =
is_started_on_replica ?
server_version >= 150000 ?
(is_started_on_replica ?
stop_backup_on_replica_after15_query :
stop_backup_on_master_after15_query
) :
(is_started_on_replica ?
stop_backup_on_replica_query :
stop_backup_on_master_query;
stop_backup_on_master_query
);
bool sent = false;
/* Make proper timestamp format for parse_time(recovery_time) */
@@ -1606,7 +1647,11 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica
*/
sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING);
if (!sent)
#if PG_VERSION_NUM >= 150000
elog(ERROR, "Failed to send pg_backup_stop query");
#else
elog(ERROR, "Failed to send pg_stop_backup query");
#endif
/* After we have sent pg_stop_backup, we don't need this callback anymore */
pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_params);
@@ -1652,7 +1697,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version,
if (interrupted)
{
pgut_cancel(conn);
#if PG_VERSION_NUM >= 150000
elog(ERROR, "interrupted during waiting for pg_backup_stop");
#else
elog(ERROR, "interrupted during waiting for pg_stop_backup");
#endif
}
if (pg_stop_backup_timeout == 1)
@@ -1665,7 +1714,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version,
if (pg_stop_backup_timeout > timeout)
{
pgut_cancel(conn);
#if PG_VERSION_NUM >= 150000
elog(ERROR, "pg_backup_stop doesn't answer in %d seconds, cancel it", timeout);
#else
elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout);
#endif
}
}
else
@@ -1677,7 +1730,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version,
/* Check successfull execution of pg_stop_backup() */
if (!query_result)
#if PG_VERSION_NUM >= 150000
elog(ERROR, "pg_backup_stop() failed");
#else
elog(ERROR, "pg_stop_backup() failed");
#endif
else
{
switch (PQresultStatus(query_result))
src/checkdb.c
+7 -3
View File
@@ -461,7 +461,9 @@ get_index_list(const char *dbname, bool first_db_with_amcheck,
"LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid "
"LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid "
"LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid "
"WHERE am.amname='btree' AND cls.relpersistence != 't' "
"WHERE am.amname='btree' "
"AND cls.relpersistence != 't' "
"AND cls.relkind != 'I' "
"ORDER BY nmspc.nspname DESC",
0, NULL);
}
@@ -473,8 +475,10 @@ get_index_list(const char *dbname, bool first_db_with_amcheck,
"LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid "
"LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid "
"LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid "
"WHERE am.amname='btree' AND cls.relpersistence != 't' AND "
"(cls.reltablespace IN "
"WHERE am.amname='btree' "
"AND cls.relpersistence != 't' "
"AND cls.relkind != 'I' "
"AND (cls.reltablespace IN "
"(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') "
"OR cls.reltablespace = 0) "
"ORDER BY nmspc.nspname DESC",
src/configure.c
+58 -58
View File
@@ -55,7 +55,7 @@ ConfigOption instance_options[] =
/* Instance options */
{
's', 'D', "pgdata",
&instance_config.pgdata, SOURCE_CMD, 0,
&instance_config.pgdata, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_INSTANCE_GROUP, 0, option_get_value
},
{
@@ -72,28 +72,28 @@ ConfigOption instance_options[] =
#endif
{
's', 'E', "external-dirs",
&instance_config.external_dir_str, SOURCE_CMD, 0,
&instance_config.external_dir_str, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_INSTANCE_GROUP, 0, option_get_value
},
/* Connection options */
{
's', 'd', "pgdatabase",
&instance_config.conn_opt.pgdatabase, SOURCE_CMD, 0,
&instance_config.conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_CONN_GROUP, 0, option_get_value
},
{
's', 'h', "pghost",
&instance_config.conn_opt.pghost, SOURCE_CMD, 0,
&instance_config.conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_CONN_GROUP, 0, option_get_value
},
{
's', 'p', "pgport",
&instance_config.conn_opt.pgport, SOURCE_CMD, 0,
&instance_config.conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_CONN_GROUP, 0, option_get_value
},
{
's', 'U', "pguser",
&instance_config.conn_opt.pguser, SOURCE_CMD, 0,
&instance_config.conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_CONN_GROUP, 0, option_get_value
},
/* Obsolete options */
@@ -125,17 +125,17 @@ ConfigOption instance_options[] =
},
{
's', 208, "archive-host",
&instance_config.archive.host, SOURCE_CMD, 0,
&instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_ARCHIVE_GROUP, 0, option_get_value
},
{
's', 209, "archive-port",
&instance_config.archive.port, SOURCE_CMD, 0,
&instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_ARCHIVE_GROUP, 0, option_get_value
},
{
's', 210, "archive-user",
&instance_config.archive.user, SOURCE_CMD, 0,
&instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_ARCHIVE_GROUP, 0, option_get_value
},
{
@@ -146,37 +146,37 @@ ConfigOption instance_options[] =
/* Logging options */
{
'f', 212, "log-level-console",
assign_log_level_console, SOURCE_CMD, 0,
assign_log_level_console, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, get_log_level_console
},
{
'f', 213, "log-level-file",
assign_log_level_file, SOURCE_CMD, 0,
assign_log_level_file, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, get_log_level_file
},
{
'f', 214, "log-format-console",
assign_log_format_console, SOURCE_CMD_STRICT, 0,
assign_log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, get_log_format_console
},
{
'f', 215, "log-format-file",
assign_log_format_file, SOURCE_CMD, 0,
assign_log_format_file, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, get_log_format_file
},
{
's', 216, "log-filename",
&instance_config.logger.log_filename, SOURCE_CMD, 0,
&instance_config.logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
's', 217, "error-log-filename",
&instance_config.logger.error_log_filename, SOURCE_CMD, 0,
&instance_config.logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
's', 218, "log-directory",
&instance_config.logger.log_directory, SOURCE_CMD, 0,
&instance_config.logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
@@ -192,64 +192,64 @@ ConfigOption instance_options[] =
/* Retention options */
{
'u', 221, "retention-redundancy",
&instance_config.retention_redundancy, SOURCE_CMD, 0,
&instance_config.retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_RETENTION_GROUP, 0, option_get_value
},
{
'u', 222, "retention-window",
&instance_config.retention_window, SOURCE_CMD, 0,
&instance_config.retention_window, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_RETENTION_GROUP, 0, option_get_value
},
{
'u', 223, "wal-depth",
&instance_config.wal_depth, SOURCE_CMD, 0,
&instance_config.wal_depth, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_RETENTION_GROUP, 0, option_get_value
},
/* Compression options */
{
'f', 224, "compress-algorithm",
assign_compress_alg, SOURCE_CMD, 0,
assign_compress_alg, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_COMPRESS_GROUP, 0, get_compress_alg
},
{
'u', 225, "compress-level",
&instance_config.compress_level, SOURCE_CMD, 0,
&instance_config.compress_level, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_COMPRESS_GROUP, 0, option_get_value
},
/* Remote backup options */
{
's', 226, "remote-proto",
&instance_config.remote.proto, SOURCE_CMD, 0,
&instance_config.remote.proto, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 227, "remote-host",
&instance_config.remote.host, SOURCE_CMD, 0,
&instance_config.remote.host, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 228, "remote-port",
&instance_config.remote.port, SOURCE_CMD, 0,
&instance_config.remote.port, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 229, "remote-path",
&instance_config.remote.path, SOURCE_CMD, 0,
&instance_config.remote.path, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 230, "remote-user",
&instance_config.remote.user, SOURCE_CMD, 0,
&instance_config.remote.user, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 231, "ssh-options",
&instance_config.remote.ssh_options, SOURCE_CMD, 0,
&instance_config.remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 232, "ssh-config",
&instance_config.remote.ssh_config, SOURCE_CMD, 0,
&instance_config.remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{ 0 }
@@ -407,7 +407,7 @@ readInstanceConfigFile(InstanceState *instanceState)
/* Instance options */
{
's', 'D', "pgdata",
&instance->pgdata, SOURCE_CMD, 0,
&instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_INSTANCE_GROUP, 0, option_get_value
},
{
@@ -424,28 +424,28 @@ readInstanceConfigFile(InstanceState *instanceState)
#endif
{
's', 'E', "external-dirs",
&instance->external_dir_str, SOURCE_CMD, 0,
&instance->external_dir_str, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_INSTANCE_GROUP, 0, option_get_value
},
/* Connection options */
{
's', 'd', "pgdatabase",
&instance->conn_opt.pgdatabase, SOURCE_CMD, 0,
&instance->conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_CONN_GROUP, 0, option_get_value
},
{
's', 'h', "pghost",
&instance->conn_opt.pghost, SOURCE_CMD, 0,
&instance->conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_CONN_GROUP, 0, option_get_value
},
{
's', 'p', "pgport",
&instance->conn_opt.pgport, SOURCE_CMD, 0,
&instance->conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_CONN_GROUP, 0, option_get_value
},
{
's', 'U', "pguser",
&instance->conn_opt.pguser, SOURCE_CMD, 0,
&instance->conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_CONN_GROUP, 0, option_get_value
},
/* Archive options */
@@ -456,66 +456,66 @@ readInstanceConfigFile(InstanceState *instanceState)
},
{
's', 208, "archive-host",
&instance_config.archive.host, SOURCE_CMD, 0,
&instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_ARCHIVE_GROUP, 0, option_get_value
},
{
's', 209, "archive-port",
&instance_config.archive.port, SOURCE_CMD, 0,
&instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_ARCHIVE_GROUP, 0, option_get_value
},
{
's', 210, "archive-user",
&instance_config.archive.user, SOURCE_CMD, 0,
&instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_ARCHIVE_GROUP, 0, option_get_value
},
{
's', 211, "restore-command",
&instance->restore_command, SOURCE_CMD, 0,
&instance->restore_command, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_ARCHIVE_GROUP, 0, option_get_value
},
/* Instance options */
{
's', 'D', "pgdata",
&instance->pgdata, SOURCE_CMD, 0,
&instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_INSTANCE_GROUP, 0, option_get_value
},
/* Logging options */
{
's', 212, "log-level-console",
&log_level_console, SOURCE_CMD, 0,
&log_level_console, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
's', 213, "log-level-file",
&log_level_file, SOURCE_CMD, 0,
&log_level_file, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
's', 214, "log-format-console",
&log_format_console, SOURCE_CMD_STRICT, 0,
&log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
's', 215, "log-format-file",
&log_format_file, SOURCE_CMD, 0,
&log_format_file, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
's', 216, "log-filename",
&instance->logger.log_filename, SOURCE_CMD, 0,
&instance->logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
's', 217, "error-log-filename",
&instance->logger.error_log_filename, SOURCE_CMD, 0,
&instance->logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
's', 218, "log-directory",
&instance->logger.log_directory, SOURCE_CMD, 0,
&instance->logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
@@ -531,64 +531,64 @@ readInstanceConfigFile(InstanceState *instanceState)
/* Retention options */
{
'u', 221, "retention-redundancy",
&instance->retention_redundancy, SOURCE_CMD, 0,
&instance->retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_RETENTION_GROUP, 0, option_get_value
},
{
'u', 222, "retention-window",
&instance->retention_window, SOURCE_CMD, 0,
&instance->retention_window, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_RETENTION_GROUP, 0, option_get_value
},
{
'u', 223, "wal-depth",
&instance->wal_depth, SOURCE_CMD, 0,
&instance->wal_depth, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_RETENTION_GROUP, 0, option_get_value
},
/* Compression options */
{
's', 224, "compress-algorithm",
&compress_alg, SOURCE_CMD, 0,
&compress_alg, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_LOG_GROUP, 0, option_get_value
},
{
'u', 225, "compress-level",
&instance->compress_level, SOURCE_CMD, 0,
&instance->compress_level, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_COMPRESS_GROUP, 0, option_get_value
},
/* Remote backup options */
{
's', 226, "remote-proto",
&instance->remote.proto, SOURCE_CMD, 0,
&instance->remote.proto, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 227, "remote-host",
&instance->remote.host, SOURCE_CMD, 0,
&instance->remote.host, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 228, "remote-port",
&instance->remote.port, SOURCE_CMD, 0,
&instance->remote.port, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 229, "remote-path",
&instance->remote.path, SOURCE_CMD, 0,
&instance->remote.path, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 230, "remote-user",
&instance->remote.user, SOURCE_CMD, 0,
&instance->remote.user, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 231, "ssh-options",
&instance->remote.ssh_options, SOURCE_CMD, 0,
&instance->remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{
's', 232, "ssh-config",
&instance->remote.ssh_config, SOURCE_CMD, 0,
&instance->remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT,
OPTION_REMOTE_GROUP, 0, option_get_value
},
{ 0 }
src/parsexlog.c
+19 -1
View File
@@ -29,8 +29,13 @@
* RmgrNames is an array of resource manager names, to make error messages
* a bit nicer.
*/
#if PG_VERSION_NUM >= 150000
#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \
name,
#else
#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
name,
#endif
static const char *RmgrNames[RM_MAX_ID + 1] = {
#include "access/rmgrlist.h"
@@ -1764,7 +1769,12 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data,
/* Is this a special record type that I recognize? */
if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
if (rmid == RM_DBASE_ID
#if PG_VERSION_NUM >= 150000
&& (rminfo == XLOG_DBASE_CREATE_WAL_LOG || rminfo == XLOG_DBASE_CREATE_FILE_COPY))
#else
&& rminfo == XLOG_DBASE_CREATE)
#endif
{
/*
* New databases can be safely ignored. They would be completely
@@ -1818,13 +1828,21 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data,
RmgrNames[rmid], info);
}
#if PG_VERSION_NUM >= 150000
for (block_id = 0; block_id <= record->record->max_block_id; block_id++)
#else
for (block_id = 0; block_id <= record->max_block_id; block_id++)
#endif
{
RelFileNode rnode;
ForkNumber forknum;
BlockNumber blkno;
#if PG_VERSION_NUM >= 150000
if (!XLogRecGetBlockTagExtended(record, block_id, &rnode, &forknum, &blkno, NULL))
#else
if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
#endif
continue;
/* We only care about the main fork; others are copied as is */
src/pg_probackup.h
+11 -1
View File
@@ -47,6 +47,12 @@
#error Windows port requires compilation in MinGW64 UCRT environment
#endif
#if PG_VERSION_NUM >= 150000
// _() is explicitly undefined in libpq-int.h
// https://github.com/postgres/postgres/commit/28ec316787674dd74d00b296724a009b6edc2fb0
#define _(s) gettext(s)
#endif
/* Wrap the code that we're going to delete after refactoring in this define*/
#define REFACTORE_ME
@@ -838,6 +844,11 @@ extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli);
extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
IncrRestoreMode incremental_mode);
/* in remote.c */
extern void check_remote_agent_compatibility(int agent_version,
char *compatibility_str, size_t compatibility_str_max_size);
extern size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size);
/* in merge.c */
extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync);
extern void merge_backups(pgBackup *backup, pgBackup *next_backup);
@@ -1124,7 +1135,6 @@ extern uint64 get_system_identifier(fio_location location, const char *pgdata_pa
extern uint64 get_remote_system_identifier(PGconn *conn);
extern uint32 get_data_checksum_version(bool safe);
extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);
extern DBState get_system_dbstate(fio_location location, const char *pgdata_path);
extern uint32 get_xlog_seg_size(const char *pgdata_path);
extern void get_redo(fio_location location, const char *pgdata_path, RedoParams *redo);
extern void set_min_recovery_point(pgFile *file, const char *backup_path,
src/stream.c
+8
View File
@@ -260,11 +260,19 @@ StreamLog(void *arg)
ctl.synchronous = false;
ctl.mark_done = false;
#if PG_VERSION_NUM >= 150000
ctl.walmethod = CreateWalDirectoryMethod(
stream_arg->basedir,
PG_COMPRESSION_NONE,
0,
false);
#else /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 150000 */
ctl.walmethod = CreateWalDirectoryMethod(
stream_arg->basedir,
// (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level,
0,
false);
#endif /* PG_VERSION_NUM >= 150000 */
ctl.replication_slot = replication_slot;
ctl.stop_socket = PGINVALID_SOCKET;
ctl.do_sync = false; /* We sync all files at the end of backup */
src/util.c
-17
View File
@@ -311,23 +311,6 @@ get_pgcontrol_checksum(const char *pgdata_path)
return ControlFile.crc;
}
/* unused function */
DBState
get_system_dbstate(fio_location location, const char *pgdata_path)
{
ControlFileData ControlFile;
char *buffer;
size_t size;
buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, &size, false);
if (buffer == NULL)
return 0;
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
return ControlFile.state;
}
void
get_redo(fio_location location, const char *pgdata_path, RedoParams *redo)
{
src/utils/configuration.c
+1
View File
@@ -22,6 +22,7 @@
#include <pwd.h>
#endif
#include <time.h>
#include <pwd.h>
#define MAXPG_LSNCOMPONENT 8
src/utils/file.c
+16 -4
View File
@@ -232,8 +232,8 @@ fio_write_all(int fd, void const* buf, size_t size)
}
/* Get version of remote agent */
int
fio_get_agent_version(void)
void
fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size)
{
fio_header hdr;
hdr.cop = FIO_AGENT_VERSION;
@@ -241,8 +241,13 @@ fio_get_agent_version(void)
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
if (hdr.size > payload_buf_size)
{
elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size);
}
return hdr.arg;
*protocol = hdr.arg;
IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size);
}
/* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */
@@ -268,7 +273,7 @@ fio_open_stream(fio_location location, const char* path)
IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size);
#ifdef WIN32
f = tmpfile();
IO_CHECK(fwrite(f, 1, hdr.size, fio_stdin_buffer), hdr.size);
IO_CHECK(fwrite(fio_stdin_buffer, 1, hdr.size, f), hdr.size);
SYS_CHECK(fseek(f, 0, SEEK_SET));
#else
f = fmemopen(fio_stdin_buffer, hdr.size, "r");
@@ -3263,9 +3268,16 @@ fio_communicate(int in, int out)
IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size);
break;
case FIO_AGENT_VERSION:
{
size_t payload_size = prepare_compatibility_str(buf, buf_size);
hdr.arg = AGENT_PROTOCOL_VERSION;
hdr.size = payload_size;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(out, buf, payload_size), payload_size);
break;
}
case FIO_STAT: /* Get information about file with specified path */
hdr.size = sizeof(st);
rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st);
src/utils/file.h
+1 -1
View File
@@ -99,7 +99,6 @@ extern bool fio_is_remote_simple(fio_location location);
extern void fio_communicate(int in, int out);
extern void fio_disconnect(void);
extern int fio_get_agent_version(void);
#define FIO_FDMAX 64
#define FIO_PIPE_MARKER 0x40000000
@@ -113,6 +112,7 @@ extern void fio_error(int rc, int size, const char* file, int line);
#define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0)
#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0)
extern void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size);
/* fd-style functions */
extern int fio_open(fio_location location, const char* name, int mode);
src/utils/remote.c
+125 -5
View File
@@ -225,7 +225,7 @@ bool launch_agent(void)
return false;
} else {
#endif
elog(LOG, "Start SSH client process, pid %d", child_pid);
elog(LOG, "Start SSH client process, pid %d, cmd \"%s\"", child_pid, cmd);
SYS_CHECK(close(infd[1])); /* These are being used by the child */
SYS_CHECK(close(outfd[0]));
SYS_CHECK(close(errfd[1]));
@@ -234,10 +234,114 @@ bool launch_agent(void)
fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */
}
/* Make sure that remote agent has the same version
* TODO: we must also check PG version and fork edition
/* Make sure that remote agent has the same version, fork and other features to be binary compatible */
{
char payload_buf[1024];
fio_get_agent_version(&agent_version, payload_buf, sizeof payload_buf);
check_remote_agent_compatibility(agent_version, payload_buf, sizeof payload_buf);
}
return true;
}
#ifdef PGPRO_EDITION
/* PGPRO 10-13 checks to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */
static bool check_certified()
{
return strstr(PGPRO_VERSION_STR, "(certified)") ||
strstr(PGPRO_VERSION_STR, "(standard certified)");
}
#endif
static char* extract_pg_edition_str()
{
static char *vanilla = "vanilla";
#ifdef PGPRO_EDITION
static char *_1C = "1C";
static char *std = "standard";
static char *ent = "enterprise";
static char *std_cert = "standard-certified";
static char *ent_cert = "enterprise-certified";
if (strcmp(PGPRO_EDITION, _1C) == 0)
return vanilla;
if (PG_VERSION_NUM < 100000)
return PGPRO_EDITION;
/* these "certified" checks are applicable to PGPRO from 10 up to 12 versions.
* 13+ certified versions are compatible to non-certified ones */
if (PG_VERSION_NUM < 130000 && check_certified())
{
if (strcmp(PGPRO_EDITION, std) == 0)
return std_cert;
else if (strcmp(PGPRO_EDITION, ent) == 0)
return ent_cert;
else
Assert("Bad #define PGPRO_EDITION value" == 0);
}
return PGPRO_EDITION;
#else
return vanilla;
#endif
}
#define COMPATIBILITY_VAL_STR(macro) { #macro, macro, 0 }
#define COMPATIBILITY_VAL_INT(macro) { #macro, NULL, macro }
#define COMPATIBILITY_VAL_SEPARATOR "="
#define COMPATIBILITY_LINE_SEPARATOR "\n"
/*
* Compose compatibility string to be sent by pg_probackup agent
* through ssh and to be verified by pg_probackup peer.
* Compatibility string contains postgres essential vars as strings
* in format "var_name" + COMPATIBILITY_VAL_SEPARATOR + "var_value" + COMPATIBILITY_LINE_SEPARATOR
*/
agent_version = fio_get_agent_version();
size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size)
{
typedef struct compatibility_param_tag {
const char* name;
const char* strval;
int intval;
} compatibility_param;
compatibility_param compatibility_params[] = {
COMPATIBILITY_VAL_STR(PG_MAJORVERSION),
{ "edition", extract_pg_edition_str(), 0 },
COMPATIBILITY_VAL_INT(SIZEOF_VOID_P),
};
size_t result_size = 0;
int i;
*compatibility_buf = '\0';
for (i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++)
{
if (compatibility_params[i].strval != NULL)
result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size,
"%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR,
compatibility_params[i].name,
compatibility_params[i].strval);
else
result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size,
"%s" COMPATIBILITY_VAL_SEPARATOR "%d" COMPATIBILITY_LINE_SEPARATOR,
compatibility_params[i].name,
compatibility_params[i].intval);
Assert(result_size < compatibility_buf_size);
}
return result_size + 1;
}
/*
* Check incoming remote agent's compatibility params for equality to local ones.
*/
void check_remote_agent_compatibility(int agent_version, char *compatibility_str, size_t compatibility_str_max_size)
{
elog(LOG, "Agent version=%d\n", agent_version);
if (agent_version != AGENT_PROTOCOL_VERSION)
{
char agent_version_str[1024];
@@ -251,5 +355,21 @@ bool launch_agent(void)
agent_version_str, AGENT_PROTOCOL_VERSION_STR);
}
return true;
/* checking compatibility params */
if (strnlen(compatibility_str, compatibility_str_max_size) == compatibility_str_max_size)
{
elog(ERROR, "Corrupted remote compatibility protocol: compatibility string has no terminating \\0");
}
elog(LOG, "Agent compatibility params:\n%s", compatibility_str);
{
char buf[1024];
prepare_compatibility_str(buf, sizeof buf);
if(strcmp(compatibility_str, buf))
{
elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s", buf, compatibility_str);
}
}
}
tests/archive.py
+7
View File
@@ -245,6 +245,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
"--log-level-file=LOG"],
gdb=True)
# Attention! this breakpoint has been set on internal probackup function, not on a postgres core one
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
@@ -303,6 +304,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
"--log-level-file=info"],
gdb=True)
# Attention! this breakpoint has been set on internal probackup function, not on a postgres core one
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
@@ -330,9 +332,14 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
with open(log_file, 'r') as f:
log_content = f.read()
if self.get_version(node) < 150000:
self.assertIn(
"ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it",
log_content)
else:
self.assertIn(
"ERROR: pg_backup_stop doesn't answer in 60 seconds, cancel it",
log_content)
log_file = os.path.join(node.logs_dir, 'postgresql.log')
with open(log_file, 'r') as f:
tests/auth_test.py
+53
View File
@@ -51,16 +51,29 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase):
1, 0,
"Expecting Error due to missing grant on EXECUTE.")
except ProbackupException as e:
if self.get_version(node) < 150000:
self.assertIn(
"ERROR: query failed: ERROR: permission denied "
"for function pg_start_backup", e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
else:
self.assertIn(
"ERROR: query failed: ERROR: permission denied "
"for function pg_backup_start", e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
if self.get_version(node) < 150000:
node.safe_psql(
"postgres",
"GRANT EXECUTE ON FUNCTION"
" pg_start_backup(text, boolean, boolean) TO backup;")
else:
node.safe_psql(
"postgres",
"GRANT EXECUTE ON FUNCTION"
" pg_backup_start(text, boolean) TO backup;")
node.safe_psql(
'postgres',
@@ -92,16 +105,37 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase):
1, 0,
"Expecting Error due to missing grant on EXECUTE.")
except ProbackupException as e:
if self.get_version(node) < 150000:
self.assertIn(
"ERROR: query failed: ERROR: permission denied "
"for function pg_stop_backup", e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
else:
self.assertIn(
"ERROR: query failed: ERROR: permission denied "
"for function pg_backup_stop", e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
if self.get_vestion(node) < self.version_to_num('15.0'):
node.safe_psql(
"postgres",
"GRANT EXECUTE ON FUNCTION "
"pg_stop_backup(boolean, boolean) TO backup")
# Do this for ptrack backups
node.safe_psql(
"postgres",
"GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup")
else:
node.safe_psql(
"postgres",
"GRANT EXECUTE ON FUNCTION "
"pg_backup_stop(boolean) TO backup")
# Do this for ptrack backups
node.safe_psql(
"postgres",
"GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup")
self.backup_node(
backup_dir, 'node', node, options=['-U', 'backup'])
@@ -163,6 +197,7 @@ class AuthTest(unittest.TestCase):
except StartNodeException:
raise unittest.skip("Node hasn't started")
if cls.pb.get_version(cls.node) < 150000:
cls.node.safe_psql(
"postgres",
"CREATE ROLE backup WITH LOGIN PASSWORD 'password'; "
@@ -170,11 +205,29 @@ class AuthTest(unittest.TestCase):
"GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; "
"GRANT EXECUTE ON FUNCTION txid_current() TO backup; "
"GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;")
else:
cls.node.safe_psql(
"postgres",
"CREATE ROLE backup WITH LOGIN PASSWORD 'password'; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; "
"GRANT EXECUTE ON FUNCTION txid_current() TO backup; "
"GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;")
cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass')
@classmethod
tests/backup.py
+137 -6
View File
@@ -1853,7 +1853,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"CREATE SCHEMA ptrack; "
"CREATE EXTENSION ptrack WITH SCHEMA ptrack")
# PG >= 10
if self.get_version(node) < 150000:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
@@ -1890,6 +1890,44 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
# >= 15
else:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
if self.ptrack:
node.safe_psql(
@@ -2161,7 +2199,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
'backupdb',
'CREATE EXTENSION ptrack')
# PG >= 10
if self.get_version(node) < 150000:
node.safe_psql(
'backupdb',
"BEGIN; "
@@ -2179,6 +2217,25 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; "
"COMMIT;"
)
# >= 15
else:
node.safe_psql(
'backupdb',
"BEGIN; "
"CREATE ROLE backup WITH LOGIN; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; "
"COMMIT;"
)
# enable STREAM backup
node.safe_psql(
@@ -2904,7 +2961,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
'postgres',
'CREATE DATABASE backupdb')
# PG >= 10
if self.get_version(node) < 150000:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
@@ -2926,11 +2983,35 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
# >= 15
else:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
if ProbackupTest.enterprise:
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup")
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;")
sleep(2)
replica.promote()
@@ -2991,7 +3072,8 @@ class BackupTest(ProbackupTest, unittest.TestCase):
'postgres',
'CREATE DATABASE backupdb')
# PG >= 10
# >= 10 && < 15
if self.get_version(node) >= 100000 and self.get_version(node) < 150000:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
@@ -3013,11 +3095,35 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
# > 15
else:
node.safe_psql(
'backupdb',
"CREATE ROLE backup WITH LOGIN; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
if ProbackupTest.enterprise:
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup")
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;")
replica.promote()
@@ -3136,9 +3242,14 @@ class BackupTest(ProbackupTest, unittest.TestCase):
'WARNING: backup in progress, stop backup',
log_content)
if self.get_version(node) < 150000:
self.assertIn(
'FROM pg_catalog.pg_stop_backup',
log_content)
else:
self.assertIn(
'FROM pg_catalog.pg_backup_stop',
log_content)
self.assertIn(
'setting its status to ERROR',
@@ -3167,9 +3278,14 @@ class BackupTest(ProbackupTest, unittest.TestCase):
self.simple_bootstrap(node, 'backup')
if self.get_version(node) < 150000:
node.safe_psql(
'postgres',
'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup')
else:
node.safe_psql(
'postgres',
'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) FROM backup')
# Full backup in streaming mode
try:
@@ -3177,17 +3293,32 @@ class BackupTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node,
options=['--stream', '-U', 'backup'])
# we should die here because exception is what we expect to happen
if self.get_version(node) < 150000:
self.assertEqual(
1, 0,
"Expecting Error because of missing permissions on pg_stop_backup "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
else:
self.assertEqual(
1, 0,
"Expecting Error because of missing permissions on pg_backup_stop "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
if self.get_version(node) < 150000:
self.assertIn(
"ERROR: permission denied for function pg_stop_backup",
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
else:
self.assertIn(
"ERROR: permission denied for function pg_backup_stop",
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
self.assertIn(
"query was: SELECT pg_catalog.txid_snapshot_xmax",
e.message,
tests/catchup.py
+4
View File
@@ -1361,6 +1361,7 @@ class CatchupTest(ProbackupTest, unittest.TestCase):
dst_options = {}
dst_options['port'] = str(dst_pg.port)
self.set_auto_conf(dst_pg, dst_options)
dst_pg._assign_master(src_pg)
dst_pg.slow_start(replica = True)
dst_pg.stop()
@@ -1389,6 +1390,7 @@ class CatchupTest(ProbackupTest, unittest.TestCase):
# check: run verification query
src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)")
src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
dst_pg.catchup() # wait for replication
dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy')
@@ -1418,6 +1420,7 @@ class CatchupTest(ProbackupTest, unittest.TestCase):
# check: run verification query
src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(2*42)")
src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
dst_pg.catchup() # wait for replication
dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy')
@@ -1446,6 +1449,7 @@ class CatchupTest(ProbackupTest, unittest.TestCase):
# check: run verification query
src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(3*42)")
src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
dst_pg.catchup() # wait for replication
dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy')
tests/checkdb.py
+9 -6
View File
@@ -39,6 +39,15 @@ class CheckdbTest(ProbackupTest, unittest.TestCase):
"postgres",
"create index on t_heap(id)")
node.safe_psql(
"postgres",
"create table idxpart (a int) "
"partition by range (a)")
node.safe_psql(
"postgres",
"create index on idxpart(a)")
try:
node.safe_psql(
"postgres",
@@ -668,18 +677,12 @@ class CheckdbTest(ProbackupTest, unittest.TestCase):
'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; '
'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;'
'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;'
'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;'
)
if ProbackupTest.enterprise:
# amcheck-1.1
node.safe_psql(
'backupdb',
'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup')
else:
# amcheck-1.0
node.safe_psql(
'backupdb',
'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup')
# >= 11 < 14
elif self.get_version(node) > 110000 and self.get_version(node) < 140000:
node.safe_psql(
tests/compatibility.py
+81
View File
@@ -8,8 +8,89 @@ import shutil
module_name = 'compatibility'
def check_manual_tests_enabled():
return 'PGPROBACKUP_MANUAL' in os.environ and os.environ['PGPROBACKUP_MANUAL'] == 'ON'
def check_ssh_agent_path_exists():
return 'PGPROBACKUP_SSH_AGENT_PATH' in os.environ
class CompatibilityTest(ProbackupTest, unittest.TestCase):
def setUp(self):
self.fname = self.id().split('.')[3]
# @unittest.expectedFailure
@unittest.skipUnless(check_manual_tests_enabled(), 'skip manual test')
@unittest.skipUnless(check_ssh_agent_path_exists(), 'skip no ssh agent path exist')
# @unittest.skip("skip")
def test_catchup_with_different_remote_major_pg(self):
"""
Decription in jira issue PBCKP-236
This test exposures ticket error using pg_probackup builds for both PGPROEE11 and PGPROEE9_6
Prerequisites:
- pg_probackup git tag for PBCKP 2.5.1
- master pg_probackup build should be made for PGPROEE11
- agent pg_probackup build should be made for PGPROEE9_6
Calling probackup PGPROEE9_6 pg_probackup agent from PGPROEE11 pg_probackup master for DELTA backup causes
the PBCKP-236 problem
Please give env variables PROBACKUP_MANUAL=ON;PGPROBACKUP_SSH_AGENT_PATH=<pg_probackup_ssh_agent_path>
for the test
Please make path for agent's pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.ee.9.6/bin/'
without pg_probackup executable
"""
self.verbose = True
self.remote = True
# please use your own local path like
# pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.clean/bin/'
pgprobackup_ssh_agent_path = os.environ['PGPROBACKUP_SSH_AGENT_PATH']
src_pg = self.make_simple_node(
base_dir=os.path.join(module_name, self.fname, 'src'),
set_replication=True,
)
src_pg.slow_start()
src_pg.safe_psql(
"postgres",
"CREATE TABLE ultimate_question AS SELECT 42 AS answer")
# do full catchup
dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
self.catchup_node(
backup_mode='FULL',
source_pgdata=src_pg.data_dir,
destination_node=dst_pg,
options=['-d', 'postgres', '-p', str(src_pg.port), '--stream']
)
dst_options = {'port': str(dst_pg.port)}
self.set_auto_conf(dst_pg, dst_options)
dst_pg.slow_start()
dst_pg.stop()
src_pg.safe_psql(
"postgres",
"CREATE TABLE ultimate_question2 AS SELECT 42 AS answer")
# do delta catchup with remote pg_probackup agent with another postgres major version
# this DELTA backup should fail without PBCKP-236 patch.
self.catchup_node(
backup_mode='DELTA',
source_pgdata=src_pg.data_dir,
destination_node=dst_pg,
# here's substitution of --remoge-path pg_probackup agent compiled with another postgres version
options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pgprobackup_ssh_agent_path]
)
# Clean after yourself
self.del_test_dir(module_name, self.fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_backward_compatibility_page(self):
tests/false_positive.py
+3
View File
@@ -195,6 +195,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
gdb = self.backup_node(backup_dir, 'node', node, gdb=True)
# Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
gdb.remove_all_breakpoints()
@@ -254,6 +255,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node,
options=['--log-level-console=LOG'], gdb=True)
# Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
gdb.remove_all_breakpoints()
@@ -305,6 +307,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, gdb=True,
options=['--stream', '--log-level-file=LOG'])
# Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
tests/helpers/ptrack_helpers.py
+18 -1
View File
@@ -446,7 +446,8 @@ class ProbackupTest(object):
'postgres',
'CREATE ROLE {0} WITH LOGIN REPLICATION'.format(role))
# PG >= 10
# >= 10 && < 15
if self.get_version(node) < 150000:
node.safe_psql(
'postgres',
'GRANT USAGE ON SCHEMA pg_catalog TO {0}; '
@@ -461,6 +462,22 @@ class ProbackupTest(object):
'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role))
# >= 15
else:
node.safe_psql(
'postgres',
'GRANT USAGE ON SCHEMA pg_catalog TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; '
'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role))
def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False):
res = node.execute(
tests/locking.py
+2
View File
@@ -419,6 +419,8 @@ class LockingTest(ProbackupTest, unittest.TestCase):
and stop it in the middle, take page backup.
Expect PAGE backup to be successfully executed
"""
self._check_gdb_flag_or_skip_test()
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
tests/ptrack.py
+40 -2
View File
@@ -513,7 +513,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
"postgres",
"CREATE DATABASE backupdb")
# PG >= 10
# PG < 15
if self.get_version(node) < 150000:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
@@ -548,6 +549,42 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
# >= 15
else:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
node.safe_psql(
"backupdb",
@@ -566,7 +603,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
if ProbackupTest.enterprise:
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup")
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; "
'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;')
self.backup_node(
backup_dir, 'node', node,
tests/replica.py
+2 -1
View File
@@ -720,6 +720,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
'--stream'],
gdb=True)
# Attention! this breakpoint is set to a probackup internal function, not a postgres core one
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
gdb.remove_all_breakpoints()
@@ -751,7 +752,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
log_content)
self.assertIn(
'LOG: stop_lsn: 0/4000000',
'INFO: stop_lsn: 0/4000000',
log_content)
self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE')
tests/restore.py
+41 -2
View File
@@ -3189,7 +3189,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
"postgres",
"CREATE DATABASE backupdb")
# PG >= 10
# PG < 15
if self.get_version(node) >= 100000 and self.get_version(node) < 150000:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
@@ -3225,6 +3226,43 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
# >= 15
else:
node.safe_psql(
'backupdb',
"REVOKE ALL ON DATABASE backupdb from PUBLIC; "
"REVOKE ALL ON SCHEMA public from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
"REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
"REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
"CREATE ROLE backup WITH LOGIN REPLICATION; "
"GRANT CONNECT ON DATABASE backupdb to backup; "
"GRANT USAGE ON SCHEMA pg_catalog TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; "
"GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
"GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
)
if self.ptrack:
# TODO why backup works without these grants ?
@@ -3241,7 +3279,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.safe_psql(
"backupdb",
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup")
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; "
"GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;")
# FULL backup without database_map
backup_id = self.backup_node(
tests/retention.py
+2
View File
@@ -1519,6 +1519,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
gdb = self.backup_node(
backup_dir, 'node', node, backup_type='page', gdb=True)
# Attention! this breakpoint has been set on internal probackup function, not on a postgres core one
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
gdb.remove_all_breakpoints()
@@ -1568,6 +1569,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
gdb = self.backup_node(
backup_dir, 'node', node, backup_type='page', gdb=True)
# Attention! this breakpoint has been set on internal probackup function, not on a postgres core one
gdb.set_breakpoint('pg_stop_backup')
gdb.run_until_break()
gdb._execute('signal SIGKILL')
tests/time_consuming.py
+18 -12
View File
@@ -15,22 +15,28 @@ class TimeConsumingTests(ProbackupTest, unittest.TestCase):
run pgbench, vacuum VERBOSE FULL and ptrack backups in parallel
"""
# init node
if self.pg_config_version < self.version_to_num('11.0'):
return unittest.skip('You need PostgreSQL >= 11 for this test')
if not self.ptrack:
return unittest.skip('Skipped because ptrack support is disabled')
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
node.append_conf('postgresql.conf',
"""
max_connections = 100
wal_keep_size = 16000
ptrack.map_size = 1
shared_preload_libraries='ptrack'
log_statement = 'none'
fsync = off
log_checkpoints = on
autovacuum = off
""")
ptrack_enable=self.ptrack,
initdb_params=['--data-checksums'],
pg_options={
'max_connections': 100,
'log_statement': 'none',
'log_checkpoints': 'on',
'autovacuum': 'off',
'ptrack.map_size': 1})
if node.major_version >= 13:
self.set_auto_conf(node, {'wal_keep_size': '16000MB'})
else:
self.set_auto_conf(node, {'wal_keep_segments': '1000'})
# init probackup and add an instance
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
tests/validate.py
+3 -3
View File
@@ -4008,9 +4008,9 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
"Output: {0} \n CMD: {1}".format(
self.output, self.cmd))
except ProbackupException as e:
self.assertTrue(
'WARNING: An error occured during metadata decompression' in e.message and
'data error' in e.message,
self.assertRegex(
e.message,
r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
travis/run_tests.sh
+1 -1
View File
@@ -47,7 +47,7 @@ cd postgres # Go to postgres dir
if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then
git apply -3 ../ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff
fi
CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls
CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls --with-python
make -s -j$(nproc) install
#make -s -j$(nproc) -C 'src/common' install
#make -s -j$(nproc) -C 'src/port' install